diff --git a/.gitignore b/.gitignore index 372e57abbf4172b06a3372221b70cac1ee6fb1ee..369f698dd80459d1d846e1eb8eccc6eda974939e 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ modules.order /vmlinux.symvers /vmlinux-gdb.py /vmlinuz +/vmlinuz.efi /System.map /Module.markers /modules.builtin.modinfo diff --git a/Documentation/admin-guide/kernel-parameters.rst b/Documentation/admin-guide/kernel-parameters.rst index 6d421694d98e9e2f528877949fc2ca1e40802650..7700d6a1cd2544696ceabb045c80e288a5ced49f 100644 --- a/Documentation/admin-guide/kernel-parameters.rst +++ b/Documentation/admin-guide/kernel-parameters.rst @@ -115,6 +115,7 @@ parameter is applicable:: KVM Kernel Virtual Machine support is enabled. LIBATA Libata driver is enabled LP Printer support is enabled. + LOONGARCH LoongArch architecture is enabled. LOOP Loopback device support is enabled. M68k M68k architecture is enabled. These options have more detailed description inside of diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4ccf4246233e3cbc2da01940163b126fa7b47c89..69c898937ca8c95494e2f1d6a07b1a2687b06b71 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6102,6 +6102,12 @@ When enabled, memory and cache locality will be impacted. + writecombine= [LOONGARCH] Control the MAT (Memory Access Type) of + ioremap_wc(). + + on - Enable writecombine, use WUC for ioremap_wc() + off - Disable writecombine, use SUC for ioremap_wc() + x2apic_phys [X86-64,APIC] Use x2apic physical mode instead of default x2apic cluster mode on platforms supporting x2apic. diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index 6b0c7b650deaae2712bb9705e318ee9469f781bf..ec5ca90f3dd40609751b4fbc0e40236b1fba78e2 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -422,8 +422,8 @@ ignore-unaligned-usertrap On architectures where unaligned accesses cause traps, and where this feature is supported (``CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN``; -currently, ``arc`` and ``ia64``), controls whether all unaligned traps -are logged. +currently, ``arc``, ``ia64`` and ``loongarch``), controls whether all +unaligned traps are logged. = ============================================================= 0 Log all unaligned accesses. @@ -1435,8 +1435,8 @@ unaligned-trap On architectures where unaligned accesses cause traps, and where this feature is supported (``CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW``; currently, -``arc`` and ``parisc``), controls whether unaligned traps are caught -and emulated (instead of failing). +``arc``, ``parisc`` and ``loongarch``), controls whether unaligned traps +are caught and emulated (instead of failing). = ======================================================== 0 Do not emulate unaligned accesses. diff --git a/Documentation/filesystems/directory-locking.rst b/Documentation/filesystems/directory-locking.rst index dccd61c7c5c3bef2f12506dd3de986e894c8edda..193c22687851a7f583cdc002ed41202e14a659f2 100644 --- a/Documentation/filesystems/directory-locking.rst +++ b/Documentation/filesystems/directory-locking.rst @@ -22,13 +22,16 @@ exclusive. 3) object removal. Locking rules: caller locks parent, finds victim, locks victim and calls the method. Locks are exclusive. -4) rename() that is _not_ cross-directory. Locking rules: caller locks the -parent and finds source and target. We lock both (provided they exist). If we -need to lock two inodes of different type (dir vs non-dir), we lock directory -first. If we need to lock two inodes of the same type, lock them in inode -pointer order. Then call the method. All locks are exclusive. -NB: we might get away with locking the source (and target in exchange -case) shared. +4) rename() that is _not_ cross-directory. Locking rules: caller locks +the parent and finds source and target. Then we decide which of the +source and target need to be locked. Source needs to be locked if it's a +non-directory; target - if it's a non-directory or about to be removed. +Take the locks that need to be taken, in inode pointer order if need +to take both (that can happen only when both source and target are +non-directories - the source because it wouldn't be locked otherwise +and the target because mixing directory and non-directory is allowed +only with RENAME_EXCHANGE, and that won't be removing the target). +After the locks had been taken, call the method. All locks are exclusive. 5) link creation. Locking rules: @@ -44,20 +47,17 @@ rules: * lock the filesystem * lock parents in "ancestors first" order. If one is not ancestor of - the other, lock them in inode pointer order. + the other, lock the parent of source first. * find source and target. * if old parent is equal to or is a descendent of target fail with -ENOTEMPTY * if new parent is equal to or is a descendent of source fail with -ELOOP - * Lock both the source and the target provided they exist. If we - need to lock two inodes of different type (dir vs non-dir), we lock - the directory first. If we need to lock two inodes of the same type, - lock them in inode pointer order. + * Lock subdirectories involved (source before target). + * Lock non-directories involved, in inode pointer order. * call the method. -All ->i_rwsem are taken exclusive. Again, we might get away with locking -the source (and target in exchange case) shared. +All ->i_rwsem are taken exclusive. The rules above obviously guarantee that all directories that are going to be read, modified or removed by method will be locked by caller. @@ -67,6 +67,7 @@ If no directory is its own ancestor, the scheme above is deadlock-free. Proof: +[XXX: will be updated once we are done massaging the lock_rename()] First of all, at any moment we have a linear ordering of the objects - A < B iff (A is an ancestor of B) or (B is not an ancestor of A and ptr(A) < ptr(B)). diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index c0f2c7586531b0a562d7d5e62f44a456989a57c2..fbd695d66905f5710e8977358869334c88b9b152 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -95,7 +95,7 @@ symlink: exclusive mkdir: exclusive unlink: exclusive (both) rmdir: exclusive (both)(see below) -rename: exclusive (all) (see below) +rename: exclusive (both parents, some children) (see below) readlink: no get_link: no setattr: exclusive @@ -113,6 +113,9 @@ tmpfile: no Additionally, ->rmdir(), ->unlink() and ->rename() have ->i_rwsem exclusive on victim. cross-directory ->rename() has (per-superblock) ->s_vfs_rename_sem. + ->unlink() and ->rename() have ->i_rwsem exclusive on all non-directories + involved. + ->rename() has ->i_rwsem exclusive on any subdirectory that changes parent. See Documentation/filesystems/directory-locking.rst for more detailed discussion of the locking scheme for directory operations. diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index b00fb6313b58b779fe87f88e0fd20c281539de19..0a2d29d844190a23f66506478422b6c9a04dd3e6 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -870,7 +870,16 @@ its result is kern_unmount() or kern_unmount_array(). **mandatory** -Calling conventions for file_open_root() changed; now it takes struct path * -instead of passing mount and dentry separately. For callers that used to -pass mnt_root> pair (i.e. the root of given mount), a new helper -is provided - file_open_root_mnt(). In-tree users adjusted. +If ->rename() update of .. on cross-directory move needs an exclusion with +directory modifications, do *not* lock the subdirectory in question in your +->rename() - it's done by the caller now [that item should've been added in +28eceeda130f "fs: Lock moved directories"]. + +--- + +**mandatory** + +On same-directory ->rename() the (tautological) update of .. is not protected +by any locks; just don't do it if the old parent is the same as the new one. +We really can't lock two subdirectories in same-directory rename - not without +deadlocks. diff --git a/MAINTAINERS b/MAINTAINERS index 52fd22591b1a0e99c71bd6e8dd6624afd53b84ea..d45732e671413ae9ec6a4bf2372773da9a68c723 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10282,6 +10282,14 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git F: drivers/hid/hid-lg-g15.c +LOONGARCH +M: Huacai Chen +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git +F: Documentation/loongson/ +F: arch/loongarch/ +F: drivers/platform/loongarch/ + LSILOGIC MPT FUSION DRIVERS (FC/SAS/SPI) M: Sathya Prakash M: Sreekanth Reddy diff --git a/Makefile b/Makefile index 49a3d47af1dacec3d25375a6eca6a836259d3472..f89230fb5506ea7c3613f11ec9cf42a3b48633af 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 PATCHLEVEL = 10 -SUBLEVEL = 208 +SUBLEVEL = 209 EXTRAVERSION = NAME = Dare mighty things diff --git a/README.OpenSource b/README.OpenSource index cf348bbc847b0ab43b16dd808a95ba64d1cc9084..7c8b594ddba607bc19d701fe50d2a2c3029aad32 100644 --- a/README.OpenSource +++ b/README.OpenSource @@ -3,7 +3,7 @@ "Name": "linux-5.10", "License": "GPL-2.0+", "License File": "COPYING", - "Version Number": "5.10.208", + "Version Number": "5.10.209", "Owner": "liuyu82@huawei.com", "Upstream URL": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/log/?h=linux-5.10.y", "Description": "linux kernel 5.10" diff --git a/arch/arm/boot/dts/exynos4210-i9100.dts b/arch/arm/boot/dts/exynos4210-i9100.dts index d186b93144e38ab9fab37ea9c8a2338eed119915..5256181971973e65ce0e78692530659720896c50 100644 --- a/arch/arm/boot/dts/exynos4210-i9100.dts +++ b/arch/arm/boot/dts/exynos4210-i9100.dts @@ -464,6 +464,14 @@ vtcam_reg: LDO12 { regulator-name = "VT_CAM_1.8V"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; + + /* + * Force-enable this regulator; otherwise the + * kernel hangs very early in the boot process + * for about 12 seconds, without apparent + * reason. + */ + regulator-always-on; }; vcclcd_reg: LDO13 { diff --git a/arch/arm/boot/dts/imx1-ads.dts b/arch/arm/boot/dts/imx1-ads.dts index 5833fb6f15d88acc096d7acb71fe518153adcd8b..2c817c4a4c68f8ec9e100db747762067c7a4b483 100644 --- a/arch/arm/boot/dts/imx1-ads.dts +++ b/arch/arm/boot/dts/imx1-ads.dts @@ -65,7 +65,7 @@ &weim { pinctrl-0 = <&pinctrl_weim>; status = "okay"; - nor: nor@0,0 { + nor: flash@0,0 { compatible = "cfi-flash"; reg = <0 0x00000000 0x02000000>; bank-width = <4>; diff --git a/arch/arm/boot/dts/imx1-apf9328.dts b/arch/arm/boot/dts/imx1-apf9328.dts index 77b21aa7a146951970d46b2943471fdc1f69e447..27e72b07b517aebcc0adb672df1e94d987a07216 100644 --- a/arch/arm/boot/dts/imx1-apf9328.dts +++ b/arch/arm/boot/dts/imx1-apf9328.dts @@ -45,7 +45,7 @@ &weim { pinctrl-0 = <&pinctrl_weim>; status = "okay"; - nor: nor@0,0 { + nor: flash@0,0 { compatible = "cfi-flash"; reg = <0 0x00000000 0x02000000>; bank-width = <2>; diff --git a/arch/arm/boot/dts/imx1.dtsi b/arch/arm/boot/dts/imx1.dtsi index 9b940987864c715297eaa5b86247969be453bfbf..8d6e900a9081e723c6176ad12b3d26d90da95c07 100644 --- a/arch/arm/boot/dts/imx1.dtsi +++ b/arch/arm/boot/dts/imx1.dtsi @@ -268,9 +268,12 @@ weim: weim@220000 { status = "disabled"; }; - esram: esram@300000 { + esram: sram@300000 { compatible = "mmio-sram"; reg = <0x00300000 0x20000>; + ranges = <0 0x00300000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; }; }; }; diff --git a/arch/arm/boot/dts/imx25-eukrea-cpuimx25.dtsi b/arch/arm/boot/dts/imx25-eukrea-cpuimx25.dtsi index 0703f62d10d1cb1d3b0d262e0539523410d3f915..93a6e4e680b45133885a7c04693ae2e49dd1db85 100644 --- a/arch/arm/boot/dts/imx25-eukrea-cpuimx25.dtsi +++ b/arch/arm/boot/dts/imx25-eukrea-cpuimx25.dtsi @@ -27,7 +27,7 @@ &i2c1 { pinctrl-0 = <&pinctrl_i2c1>; status = "okay"; - pcf8563@51 { + rtc@51 { compatible = "nxp,pcf8563"; reg = <0x51>; }; diff --git a/arch/arm/boot/dts/imx27-eukrea-cpuimx27.dtsi b/arch/arm/boot/dts/imx27-eukrea-cpuimx27.dtsi index 74110bbcd9d4f2ea217a77ece024a5eb76578f31..c7e92358487826874e74d819af17764311016d4a 100644 --- a/arch/arm/boot/dts/imx27-eukrea-cpuimx27.dtsi +++ b/arch/arm/boot/dts/imx27-eukrea-cpuimx27.dtsi @@ -33,7 +33,7 @@ &i2c1 { pinctrl-0 = <&pinctrl_i2c1>; status = "okay"; - pcf8563@51 { + rtc@51 { compatible = "nxp,pcf8563"; reg = <0x51>; }; @@ -90,7 +90,7 @@ &usbotg { &weim { status = "okay"; - nor: nor@0,0 { + nor: flash@0,0 { #address-cells = <1>; #size-cells = <1>; compatible = "cfi-flash"; diff --git a/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi b/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi index 3d10273177e9b5e7ff911a9c763a07a6398f11be..a5fdc2fd4ce5a680d393083d0136fa02931c030b 100644 --- a/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi +++ b/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi @@ -322,7 +322,7 @@ &usbotg { &weim { status = "okay"; - nor: nor@0,0 { + nor: flash@0,0 { compatible = "cfi-flash"; reg = <0 0x00000000 0x02000000>; bank-width = <2>; diff --git a/arch/arm/boot/dts/imx27.dtsi b/arch/arm/boot/dts/imx27.dtsi index 7bc132737a37586566c25208050c1aa9341c438f..8ae24c8655217303d55dd2a7242b13145e9e8589 100644 --- a/arch/arm/boot/dts/imx27.dtsi +++ b/arch/arm/boot/dts/imx27.dtsi @@ -588,6 +588,9 @@ weim: weim@d8002000 { iram: sram@ffff4c00 { compatible = "mmio-sram"; reg = <0xffff4c00 0xb400>; + ranges = <0 0xffff4c00 0xb400>; + #address-cells = <1>; + #size-cells = <1>; }; }; }; diff --git a/arch/arm/boot/dts/imx7d.dtsi b/arch/arm/boot/dts/imx7d.dtsi index b0bcfa9094a308093508f36283da16e1a9368d85..8ad3e60fd7d1c40e7f90d7b7d9016fe4a71bcda7 100644 --- a/arch/arm/boot/dts/imx7d.dtsi +++ b/arch/arm/boot/dts/imx7d.dtsi @@ -209,9 +209,6 @@ pcie: pcie@33800000 { }; &ca_funnel_in_ports { - #address-cells = <1>; - #size-cells = <0>; - port@1 { reg = <1>; ca_funnel_in_port1: endpoint { diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi index 03bde2fb9bb11b27ace25341d95346fba42d8fb7..b4cab6a2143709200a852efc2187bb15df3cea92 100644 --- a/arch/arm/boot/dts/imx7s.dtsi +++ b/arch/arm/boot/dts/imx7s.dtsi @@ -173,7 +173,11 @@ funnel@30041000 { clock-names = "apb_pclk"; ca_funnel_in_ports: in-ports { - port { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; ca_funnel_in_port0: endpoint { remote-endpoint = <&etm0_out_port>; }; @@ -769,7 +773,7 @@ csi_from_csi_mux: endpoint { }; lcdif: lcdif@30730000 { - compatible = "fsl,imx7d-lcdif", "fsl,imx28-lcdif"; + compatible = "fsl,imx7d-lcdif", "fsl,imx6sx-lcdif"; reg = <0x30730000 0x10000>; interrupts = ; clocks = <&clks IMX7D_LCDIF_PIXEL_ROOT_CLK>, @@ -1231,7 +1235,7 @@ dma_apbh: dma-controller@33000000 { gpmi: nand-controller@33002000{ compatible = "fsl,imx7d-gpmi-nand"; #address-cells = <1>; - #size-cells = <1>; + #size-cells = <0>; reg = <0x33002000 0x2000>, <0x33004000 0x4000>; reg-names = "gpmi-nand", "bch"; interrupts = ; diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi index 093567022386d46e2326ca5baf2c6a439fcfbd41..5f47b638f5327485d3bff6c023a24b18e891be37 100644 --- a/arch/arm/boot/dts/rk3036.dtsi +++ b/arch/arm/boot/dts/rk3036.dtsi @@ -336,12 +336,20 @@ hdmi: hdmi@20034000 { pinctrl-0 = <&hdmi_ctl>; status = "disabled"; - hdmi_in: port { + ports { #address-cells = <1>; #size-cells = <0>; - hdmi_in_vop: endpoint@0 { + + hdmi_in: port@0 { reg = <0>; - remote-endpoint = <&vop_out_hdmi>; + + hdmi_in_vop: endpoint { + remote-endpoint = <&vop_out_hdmi>; + }; + }; + + hdmi_out: port@1 { + reg = <1>; }; }; }; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index eea8d23683dc1c4954553ca82be4f3502d427910..39e9b3875761635184408e4ec9efb990d5c671ce 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -129,7 +129,7 @@ rtc@6f { compatible = "microchip,mcp7940x"; reg = <0x6f>; interrupt-parent = <&gpiosb>; - interrupts = <5 0>; /* GPIO2_5 */ + interrupts = <5 IRQ_TYPE_EDGE_FALLING>; /* GPIO2_5 */ }; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index eb07a882d43b3d4198838a9ddbbdc04303b4a93c..be40821dfeb9d2b6dafcdad5d6f35a0ed9fd911e 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -2688,10 +2688,10 @@ usb_1: usb@a6f8800 { <&gcc GCC_USB30_PRIM_MASTER_CLK>; assigned-clock-rates = <19200000>, <150000000>; - interrupts = , - , - , - ; + interrupts-extended = <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, + <&pdc 6 IRQ_TYPE_LEVEL_HIGH>, + <&pdc 8 IRQ_TYPE_EDGE_BOTH>, + <&pdc 9 IRQ_TYPE_EDGE_BOTH>; interrupt-names = "hs_phy_irq", "ss_phy_irq", "dm_hs_phy_irq", "dp_hs_phy_irq"; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 5c696ebf5c20c1a34325f5d5e0afe45e24f4b7ac..e3c6b05869e7f225bf03221dc2372baef33460e2 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -3565,10 +3565,10 @@ usb_1: usb@a6f8800 { <&gcc GCC_USB30_PRIM_MASTER_CLK>; assigned-clock-rates = <19200000>, <150000000>; - interrupts = , - , - , - ; + interrupts-extended = <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, + <&intc GIC_SPI 486 IRQ_TYPE_LEVEL_HIGH>, + <&pdc_intc 8 IRQ_TYPE_EDGE_BOTH>, + <&pdc_intc 9 IRQ_TYPE_EDGE_BOTH>; interrupt-names = "hs_phy_irq", "ss_phy_irq", "dm_hs_phy_irq", "dp_hs_phy_irq"; @@ -3613,10 +3613,10 @@ usb_2: usb@a8f8800 { <&gcc GCC_USB30_SEC_MASTER_CLK>; assigned-clock-rates = <19200000>, <150000000>; - interrupts = , - , - , - ; + interrupts-extended = <&intc GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>, + <&intc GIC_SPI 487 IRQ_TYPE_LEVEL_HIGH>, + <&pdc_intc 10 IRQ_TYPE_EDGE_BOTH>, + <&pdc_intc 11 IRQ_TYPE_EDGE_BOTH>; interrupt-names = "hs_phy_irq", "ss_phy_irq", "dm_hs_phy_irq", "dp_hs_phy_irq"; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index cdb3d4549b3a97c45bcb8de79a5f864b09e3d40f..8e428f8dd108b0fdb564e28926d3ae88d272280a 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -171,7 +171,11 @@ armv8pmu_events_sysfs_show(struct device *dev, }).attr.attr) static struct attribute *armv8_pmuv3_event_attrs[] = { - ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR), + /* + * Don't expose the sw_incr event in /sys. It's not usable as writes to + * PMSWINC_EL0 will trap as PMUSERENR.{SW,EN}=={0,0} and event rotation + * means we don't have a fixed event<->counter relationship regardless. + */ ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL), ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL), ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL), diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 23710bf5a86b6dddbb3bf7caa4e8748f8fb8b402..62f261b8eb62f69356e4e818b19f1238a3ec7bc5 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -584,7 +584,11 @@ static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db, unsigned long flags; raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); + irq = __vgic_its_check_cache(dist, db, devid, eventid); + if (irq) + vgic_get_irq_kref(irq); + raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); return irq; @@ -763,6 +767,7 @@ int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi) raw_spin_lock_irqsave(&irq->irq_lock, flags); irq->pending_latch = true; vgic_queue_irq_unlock(kvm, irq, flags); + vgic_put_irq(kvm, irq); return 0; } diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 15a6c98ee92f05cdf35f19c5ed5cbd7a6738f699..7a6360eba32143ced8ad92bed62cdde9736d8752 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -356,19 +356,26 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (test_bit(i, &val)) { - /* - * pending_latch is set irrespective of irq type - * (level or edge) to avoid dependency that VM should - * restore irq config before pending info. - */ - irq->pending_latch = true; - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); - } else { + + /* + * pending_latch is set irrespective of irq type + * (level or edge) to avoid dependency that VM should + * restore irq config before pending info. + */ + irq->pending_latch = test_bit(i, &val); + + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); irq->pending_latch = false; - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } + if (irq->pending_latch) + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + else + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); } diff --git a/arch/loongarch/.gitignore b/arch/loongarch/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e9c2dedc1c5ec1d8da8efefa2bca25363acc4223 --- /dev/null +++ b/arch/loongarch/.gitignore @@ -0,0 +1,9 @@ +*.lds +*.raw +calc_vmlinuz_load_addr +elf-entry +relocs +vmlinux.* +vmlinuz.* + +!kernel/vmlinux.lds.S diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild new file mode 100644 index 0000000000000000000000000000000000000000..7214ae22f0ec86f3aee91f0e7617d58361174376 --- /dev/null +++ b/arch/loongarch/Kbuild @@ -0,0 +1,25 @@ +# Fail on warnings - also for files referenced in subdirs +# -Werror can be disabled for specific files using: +# CFLAGS_ := -Wno-error +ifeq ($(W),) +subdir-ccflags-y := -Werror +endif + +# platform specific definitions +include arch/loongarch/Kbuild.platforms +obj-y := $(platform-y) + +# make clean traverses $(obj-) without having included .config, so +# everything ends up here +obj- := $(platform-) + +# LoongArch object files +# The object files are linked as core-y files would be linked + +obj-y += kernel/ +obj-y += mm/ +obj-y += net/ +obj-y += vdso/ + +obj-$(CONFIG_KVM) += kvm/ +obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ diff --git a/arch/loongarch/Kbuild.platforms b/arch/loongarch/Kbuild.platforms new file mode 100644 index 0000000000000000000000000000000000000000..ad390d5c00f2659915e7f2adc1d82e06fbfe920b --- /dev/null +++ b/arch/loongarch/Kbuild.platforms @@ -0,0 +1,6 @@ +# All platforms listed in alphabetic order + +platforms += loongson64 + +# include the platform specific files +include $(patsubst %, $(srctree)/arch/loongarch/%/Platform, $(platforms)) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..64cfb86477eebcea06c436285fd7bc8c3cebde03 --- /dev/null +++ b/arch/loongarch/Kconfig @@ -0,0 +1,894 @@ +# SPDX-License-Identifier: GPL-2.0 +config LOONGARCH + bool + default y + select ACPI_MCFG if ACPI + select ACPI_PPTT if ACPI + select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI + select ARCH_BINFMT_ELF_STATE + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI + select ARCH_HAS_CPU_FINALIZE_INIT + select ARCH_HAS_FORTIFY_SOURCE + select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + select ARCH_HAS_PTE_SPECIAL if !32BIT + select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST + select ARCH_INLINE_READ_LOCK if !PREEMPTION + select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION + select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_READ_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_READ_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_BH if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_WRITE_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPTION + select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPTION + select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION + select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_KEEP_MEMBLOCK + select ARCH_SUPPORTS_ACPI + select ARCH_SUPPORTS_NUMA_BALANCING + select ARCH_SUPPORTS_UPROBES + select ARCH_USE_BUILTIN_BSWAP + select ARCH_USE_CMPXCHG_LOCKREF if 64BIT + select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT + select ARCH_WANT_LD_ORPHAN_WARN + select BUILDTIME_TABLE_SORT + select COMMON_CLK + select CPU_PM + select GENERIC_ATOMIC64 if !64BIT + select GENERIC_CLOCKEVENTS + select GENERIC_CMOS_UPDATE + select GENERIC_CPU_AUTOPROBE + select GENERIC_ENTRY + select GENERIC_FIND_FIRST_BIT + select GENERIC_GETTIMEOFDAY + select GENERIC_IOREMAP if !ARCH_IOREMAP + select GENERIC_IRQ_MULTI_HANDLER + select GENERIC_IRQ_PROBE + select GENERIC_IRQ_SHOW + select GENERIC_LIB_ASHLDI3 + select GENERIC_LIB_ASHRDI3 + select GENERIC_LIB_CMPDI2 + select GENERIC_LIB_LSHRDI3 + select GENERIC_LIB_UCMPDI2 + select GENERIC_LIB_DEVMEM_IS_ALLOWED + select GENERIC_PCI_IOMAP + select GENERIC_SCHED_CLOCK + select GENERIC_SMP_IDLE_THREAD + select GENERIC_STRNCPY_FROM_USER + select GENERIC_STRNLEN_USER + select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_TIME_NS + select HANDLE_DOMAIN_IRQ + select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE + select HAVE_ARCH_KASAN if 64BIT + select HAVE_ARCH_KGDB + select HAVE_ARCH_MMAP_RND_BITS if MMU + select HAVE_ARCH_SECCOMP + select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_ASM_MODVERSIONS + select HAVE_CONTEXT_TRACKING + select HAVE_COPY_THREAD_TLS + select HAVE_C_RECORDMCOUNT + select HAVE_DEBUG_KMEMLEAK + select HAVE_DEBUG_STACKOVERFLOW + select HAVE_DMA_CONTIGUOUS + select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_EBPF_JIT if 64BIT + select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN + select HAVE_EXIT_THREAD + select HAVE_FAST_GUP + select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER + select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_GENERIC_VDSO + select HAVE_IOREMAP_PROT + select HAVE_IRQ_EXIT_ON_IRQ_STACK + select HAVE_IRQ_TIME_ACCOUNTING + select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE + select HAVE_KRETPROBES + select HAVE_LIVEPATCH + select HAVE_MEMBLOCK + select HAVE_MEMBLOCK_NODE_MAP + select HAVE_MOD_ARCH_SPECIFIC + select HAVE_NMI + select HAVE_PERF_EVENTS + select HAVE_PERF_REGS + select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_RSEQ + select HAVE_STACKPROTECTOR + select HAVE_STACK_VALIDATION + select HAVE_SYSCALL_TRACEPOINTS + select HAVE_TIF_NOHZ + select HAVE_VIRT_CPU_ACCOUNTING_GEN if 64BIT || !SMP + select IRQ_FORCED_THREADING + select MODULES_USE_ELF_RELA if MODULES + select OF + select OF_EARLY_FLATTREE + select PCI_DOMAINS_GENERIC if PCI + select PCI_ECAM if ACPI + select PCI_MSI_ARCH_FALLBACKS + select PERF_USE_VMALLOC + select RTC_LIB + select SYSCTL_ARCH_UNALIGN_ALLOW + select SYSCTL_ARCH_UNALIGN_NO_WARN + select SYSCTL_EXCEPTION_TRACE + select USER_STACKTRACE_SUPPORT + +menu "Machine selection" + +choice + prompt "System type" + default MACH_LOONGSON64 + +config MACH_LOONGSON64 + bool "Loongson 64-bit family of machines" + select ARCH_SPARSEMEM_ENABLE + select ARCH_MIGHT_HAVE_PC_PARPORT + select ARCH_MIGHT_HAVE_PC_SERIO + select HAVE_PCI + select PCI + select IRQ_LOONGARCH_CPU + select NR_CPUS_DEFAULT_64 + select SPARSE_IRQ + select SYS_HAS_CPU_LOONGSON64 + select SYS_SUPPORTS_SMP + select SYS_SUPPORTS_ZBOOT + select SYS_SUPPORTS_HOTPLUG_CPU + select SYS_SUPPORTS_NUMA + select SYS_SUPPORTS_64BIT_KERNEL + select SYS_SUPPORTS_HIGHMEM + select SYS_SUPPORTS_RELOCATABLE + select ZONE_DMA32 + select USE_OF + select BUILTIN_DTB + help + This enables the support of Loongson 64-bit family of machines. These + machines are based on new Loongson-3 processors (Old Loongson is MIPS + compatible, while new Loongson is based on LoongArch ISA). + +endchoice + +endmenu + +config GENERIC_HWEIGHT + bool + default y + +config GENERIC_CALIBRATE_DELAY + bool + default y + +config SCHED_OMIT_FRAME_POINTER + bool + default y + +config AS_HAS_EXPLICIT_RELOCS + def_bool $(as-instr,x:pcalau12i \$t0$(comma)%pc_hi20(x)) + +config ARCH_SUPPORTS_UPROBES + bool + +config SYS_SUPPORTS_HOTPLUG_CPU + bool + +config GENERIC_CSUM + def_bool y + +config SYS_SUPPORTS_RELOCATABLE + bool + help + Selected if the platform supports relocating the kernel. + +config SYS_SUPPORTS_HUGETLBFS + def_bool y + +config L1_CACHE_SHIFT + int + default "6" + +menu "CPU selection" + +choice + prompt "CPU type" + default CPU_LOONGSON64 + +config CPU_LOONGSON64 + bool "Loongson 64-bit CPU" + depends on SYS_HAS_CPU_LOONGSON64 + select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_LSX + select CPU_SUPPORTS_LASX + select GPIOLIB + select SWIOTLB + select HAVE_KVM + select ARCH_SUPPORTS_ATOMIC_RMW + help + The Loongson 64-bit processor implements the LoongArch64 (the 64-bit + version of LoongArch) instruction set. + +endchoice + +config SYS_SUPPORTS_ZBOOT + bool + select HAVE_KERNEL_GZIP + select HAVE_KERNEL_BZIP2 + select HAVE_KERNEL_LZ4 + select HAVE_KERNEL_LZMA + select HAVE_KERNEL_LZO + select HAVE_KERNEL_XZ + select HAVE_KERNEL_ZSTD + +config SYS_HAS_CPU_LOONGSON64 + bool + +endmenu + +config SYS_SUPPORTS_32BIT_KERNEL + bool +config SYS_SUPPORTS_64BIT_KERNEL + bool +config CPU_SUPPORTS_32BIT_KERNEL + bool +config CPU_SUPPORTS_64BIT_KERNEL + bool +config HARDWARE_WATCHPOINTS + bool + default y + +menu "Kernel type and options" + +choice + prompt "Kernel code model" + help + You should only select this option if you have a workload that + actually benefits from 64-bit processing or if your machine has + large memory. You will only be presented a single option in this + menu if your system does not support both 32-bit and 64-bit kernels. + +config 32BIT + bool "32-bit kernel" + depends on CPU_SUPPORTS_32BIT_KERNEL && SYS_SUPPORTS_32BIT_KERNEL + help + Select this option if you want to build a 32-bit kernel. + +config 64BIT + bool "64-bit kernel" + depends on CPU_SUPPORTS_64BIT_KERNEL && SYS_SUPPORTS_64BIT_KERNEL + help + Select this option if you want to build a 64-bit kernel. + +endchoice + +config FIX_EARLYCON_MEM + def_bool y + +config PAGE_SIZE_4KB + bool + +config PAGE_SIZE_16KB + bool + +config PAGE_SIZE_64KB + bool + +config PGTABLE_2LEVEL + bool + +config PGTABLE_3LEVEL + bool + +config PGTABLE_4LEVEL + bool + +config PGTABLE_LEVELS + int + default 2 if PGTABLE_2LEVEL + default 3 if PGTABLE_3LEVEL + default 4 if PGTABLE_4LEVEL + +choice + prompt "Page Table Layout" + default 16KB_2LEVEL if 32BIT + default 16KB_3LEVEL if 64BIT + help + Allows choosing the page table layout, which is a combination + of page size and page table levels. The size of virtual memory + address space are determined by the page table layout. + +config 4KB_3LEVEL + bool "4KB with 3 levels" + select PAGE_SIZE_4KB + select PGTABLE_3LEVEL + help + This option selects 4KB page size with 3 level page tables, which + support a maximum of 39 bits of application virtual memory. + +config 4KB_4LEVEL + bool "4KB with 4 levels" + select PAGE_SIZE_4KB + select PGTABLE_4LEVEL + help + This option selects 4KB page size with 4 level page tables, which + support a maximum of 48 bits of application virtual memory. + +config 16KB_2LEVEL + bool "16KB with 2 levels" + select PAGE_SIZE_16KB + select PGTABLE_2LEVEL + help + This option selects 16KB page size with 2 level page tables, which + support a maximum of 36 bits of application virtual memory. + +config 16KB_3LEVEL + bool "16KB with 3 levels" + select PAGE_SIZE_16KB + select PGTABLE_3LEVEL + help + This option selects 16KB page size with 3 level page tables, which + support a maximum of 47 bits of application virtual memory. + +config 64KB_2LEVEL + bool "64KB with 2 levels" + select PAGE_SIZE_64KB + select PGTABLE_2LEVEL + help + This option selects 64KB page size with 2 level page tables, which + support a maximum of 42 bits of application virtual memory. + +config 64KB_3LEVEL + bool "64KB with 3 levels" + select PAGE_SIZE_64KB + select PGTABLE_3LEVEL + help + This option selects 64KB page size with 3 level page tables, which + support a maximum of 55 bits of application virtual memory. + +endchoice + +config FORCE_MAX_ZONEORDER + int "Maximum zone order" + range 14 64 if PAGE_SIZE_64KB + default "14" if PAGE_SIZE_64KB + range 12 64 if PAGE_SIZE_16KB + default "12" if PAGE_SIZE_16KB + range 11 64 + default "11" + help + The kernel memory allocator divides physically contiguous memory + blocks into "zones", where each zone is a power of two number of + pages. This option selects the largest power of two that the kernel + keeps in the memory allocator. If you need to allocate very large + blocks of physically contiguous memory, then you may need to + increase this value. + + This config option is actually maximum order plus one. For example, + a value of 11 means that the largest free memory block is 2^10 pages. + + The page size is not necessarily 4KB. Keep this in mind + when choosing a value for this option. + +config CPU_HAS_PREFETCH + bool + default y + +config CPU_HAS_FPU + bool + default y + +config CPU_HAS_LSX + bool "Support for the Loongson SIMD Extension" + depends on CPU_SUPPORTS_LSX + depends on 64BIT + help + Loongson SIMD Extension (LSX) introduces 128 bit wide vector registers + and a set of SIMD instructions to operate on them. When this option + is enabled the kernel will support allocating & switching LSX + vector register contexts. If you know that your kernel will only be + running on CPUs which do not support LSX or that your userland will + not be making use of it then you may wish to say N here to reduce + the size & complexity of your kernel. + + If unsure, say Y. + +config CPU_HAS_LASX + bool "Support for the Loongson Advanced SIMD Extension" + depends on CPU_SUPPORTS_LASX + depends on 64BIT && CPU_HAS_LSX + help + Loongson Advanced SIMD Extension is 256 bit wide SIMD extension. + + If unsure, say Y. + +config CPU_HAS_LBT + bool "Support for the Loongson Binary Tranlatition" + +# +# - Highmem only makes sense for the 32-bit kernel. +# - We use SYS_SUPPORTS_HIGHMEM to offer highmem only for systems where we +# know they might have memory configurations that could make use of highmem +# support. +# +config HIGHMEM + bool "High Memory Support" + depends on 32BIT && CPU_SUPPORTS_HIGHMEM && SYS_SUPPORTS_HIGHMEM + +config CPU_SUPPORTS_HIGHMEM + bool + +config SYS_SUPPORTS_HIGHMEM + bool + +config CPU_SUPPORTS_LSX + bool + +config CPU_SUPPORTS_LASX + bool + +config ARCH_SELECT_MEMORY_MODEL + def_bool y + +config ARCH_FLATMEM_ENABLE + def_bool y + depends on !NUMA + +config ARCH_SPARSEMEM_ENABLE + def_bool y + select SPARSEMEM_VMEMMAP_ENABLE + help + Say Y to support efficient handling of sparse physical memory, + for architectures which are either NUMA (Non-Uniform Memory Access) + or have huge holes in the physical address space for other reasons. + See for more. + +config NUMA + bool "NUMA Support" + depends on SYS_SUPPORTS_NUMA + select ACPI_NUMA if ACPI + help + Say Y to compile the kernel with NUMA (Non-Uniform Memory Access) + support. This option improves performance on systems with more + than one NUMA node; on single node systems it is generally better + to leave it disabled. + +config SYS_SUPPORTS_NUMA + bool + +config RELOCATABLE + bool "Relocatable kernel" + depends on SYS_SUPPORTS_RELOCATABLE + help + This builds a kernel image that retains relocation information + so it can be loaded someplace besides the default 1MB. + The relocations make the kernel binary about 15% larger, + but are discarded at runtime + +config RELOCATION_TABLE_SIZE + hex "Relocation table size" + depends on RELOCATABLE + range 0x0 0x01000000 + default "0x00200000" + help + A table of relocation data will be appended to the kernel binary + and parsed at boot to fix up the relocated kernel. + + This option allows the amount of space reserved for the table to be + adjusted, although the default of 1Mb should be ok in most cases. + + The build will fail and a valid size suggested if this is too small. + + If unsure, leave at the default value. + +config RANDOMIZE_BASE + bool "Randomize the address of the kernel image" + depends on RELOCATABLE + help + Randomizes the physical and virtual address at which the + kernel image is loaded, as a security feature that + deters exploit attempts relying on knowledge of the location + of kernel internals. + + Entropy is generated using any coprocessor 0 registers available. + + The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET. + + If unsure, say N. + +config RANDOMIZE_BASE_MAX_OFFSET + hex "Maximum kASLR offset" if EXPERT + depends on RANDOMIZE_BASE + range 0x0 0x40000000 if 64BIT + range 0x0 0x08000000 + default "0x01000000" + help + When kASLR is active, this provides the maximum offset that will + be applied to the kernel image. It should be set according to the + amount of physical RAM available in the target system. + + This is limited by the size of the lower address memory, 256MB. + +config NODES_SHIFT + int + default "6" + depends on NEED_MULTIPLE_NODES + +config USE_PERCPU_NUMA_NODE_ID + def_bool y + depends on NUMA + +config HAVE_SETUP_PER_CPU_AREA + def_bool y + depends on NUMA + +config NEED_PER_CPU_EMBED_FIRST_CHUNK + def_bool y + depends on NUMA + +config NEED_PER_CPU_PAGE_FIRST_CHUNK + def_bool y + depends on NUMA + +config CMDLINE + string "Built-in kernel command line" + help + For most platforms, the arguments for the kernel's command line + are provided at run-time, during boot. However, there are cases + where either no arguments are being provided or the provided + arguments are insufficient or even invalid. + + When that occurs, it is possible to define a built-in command + line here and choose how the kernel should use it later on. + +choice + prompt "Kernel command line type" + default CMDLINE_BOOTLOADER + help + Choose how the kernel will handle the provided built-in command + line. + +config CMDLINE_BOOTLOADER + bool "Use bootloader kernel arguments if available" + help + Prefer the command-line passed by the boot loader if available. + Use the built-in command line as fallback in case we get nothing + during boot. This is the default behaviour. + +config CMDLINE_EXTEND + bool "Use built-in to extend bootloader kernel arguments" + help + The command-line arguments provided during boot will be + appended to the built-in command line. This is useful in + cases where the provided arguments are insufficient and + you don't want to or cannot modify them. + +config CMDLINE_FORCE + bool "Always use the built-in kernel command string" + help + Always use the built-in command line, even if we get one during + boot. This is useful in case you need to override the provided + command line on systems where you don't have or want control + over it. + +endchoice + +config DMI + bool "Enable DMI scanning" + select DMI_SCAN_MACHINE_NON_EFI_FALLBACK + default y + help + This enables SMBIOS/DMI feature for systems, and scanning of + DMI to identify machine quirks. + +config EFI + bool "EFI runtime service support" + select UCS2_STRING + select EFI_RUNTIME_WRAPPERS + help + This enables the kernel to use EFI runtime services that are + available (such as the EFI variable services). + +config EFI_STUB + bool "EFI stub support" + default y + depends on EFI + select EFI_GENERIC_STUB + help + This kernel feature allows a vmlinuz.efi to be loaded directly + by EFI firmware without the use of a bootloader. + +config SCHED_SMT + bool "SMT scheduler support" + default y + help + Improves scheduler's performance when there are multiple + threads in one physical core. + +config SMP + bool "Multi-Processing support" + depends on SYS_SUPPORTS_SMP + help + This enables support for systems with more than one CPU. If you have + a system with only one CPU, say N. If you have a system with more + than one CPU, say Y. + + If you say N here, the kernel will run on uni- and multiprocessor + machines, but will use only one CPU of a multiprocessor machine. If + you say Y here, the kernel will run on many, but not all, + uniprocessor machines. On a uniprocessor machine, the kernel + will run faster if you say N here. + + See also the SMP-HOWTO available at . + + If you don't know what to do here, say N. + +config HOTPLUG_CPU + bool "Support for hot-pluggable CPUs" + select GENERIC_IRQ_MIGRATION + depends on SMP && SYS_SUPPORTS_HOTPLUG_CPU + help + Say Y here to allow turning CPUs off and on. CPUs can be + controlled through /sys/devices/system/cpu. + (Note: power management support will enable this option + automatically on SMP systems. ) + Say N if you want to disable CPU hotplug. + +config SYS_SUPPORTS_SMP + bool + +config NR_CPUS_DEFAULT_4 + bool + +config NR_CPUS_DEFAULT_8 + bool + +config NR_CPUS_DEFAULT_16 + bool + +config NR_CPUS_DEFAULT_32 + bool + +config NR_CPUS_DEFAULT_64 + bool + +config NR_CPUS + int "Maximum number of CPUs (2-256)" + range 2 256 + depends on SMP + default "4" if NR_CPUS_DEFAULT_4 + default "8" if NR_CPUS_DEFAULT_8 + default "16" if NR_CPUS_DEFAULT_16 + default "32" if NR_CPUS_DEFAULT_32 + default "64" if NR_CPUS_DEFAULT_64 + help + This allows you to specify the maximum number of CPUs which this + kernel will support. The maximum supported value is 32 for 32-bit + kernel and 64 for 64-bit kernels; the minimum value which makes + sense is 1 for Qemu (useful only for kernel debugging purposes) + and 2 for all others. + + This is purely to save memory - each supported CPU adds + approximately eight kilobytes to the kernel image. For best + performance should round up your number of processors to the next + power of two. + +source "kernel/Kconfig.hz" + +config KEXEC + bool "Kexec system call" + select KEXEC_CORE + help + kexec is a system call that implements the ability to shutdown your + current kernel, and to start another kernel. It is like a reboot + but it is independent of the system firmware. And like a reboot + you can start any kernel with it, not just Linux. + + The name comes from the similarity to the exec system call. + +config CRASH_DUMP + bool "Build kdump crash kernel" + select RELOCATABLE + help + Generate crash dump after being started by kexec. This should + be normally only set in special crash dump kernels which are + loaded in the main kernel with kexec-tools into a specially + reserved region and then later executed after a crash by + kdump/kexec. + + For more details see Documentation/admin-guide/kdump/kdump.rst + +config ARCH_IOREMAP + bool "Enable LoongArch DMW-based ioremap()" + help + We use generic TLB-based ioremap() by default since it has page + protection support. However, you can enable LoongArch DMW-based + ioremap() for better performance. + +config ARCH_WRITECOMBINE + bool "Enable WriteCombine (WUC) for ioremap()" + help + LoongArch maintains cache coherency in hardware, but when paired + with LS7A chipsets the WUC attribute (Weak-ordered UnCached, which + is similar to WriteCombine) is out of the scope of cache coherency + machanism for PCIe devices (this is a PCIe protocol violation, which + may be fixed in newer chipsets). + + This means WUC can only used for write-only memory regions now, so + this option is disabled by default, making WUC silently fallback to + SUC for ioremap(). You can enable this option if the kernel is ensured + to run on hardware without this bug. + + You can override this setting via writecombine=on/off boot parameter. + +config ARCH_STRICT_ALIGN + bool "Enable -mstrict-align to prevent unaligned accesses" if EXPERT + default y + help + Not all LoongArch cores support h/w unaligned access, we can use + -mstrict-align build parameter to prevent unaligned accesses. + + CPUs with h/w unaligned access support: + Loongson-2K2000/2K3000/3A5000/3C5000/3D5000. + + CPUs without h/w unaligned access support: + Loongson-2K500/2K1000. + + This option is enabled by default to make the kernel be able to run + on all LoongArch systems. But you can disable it manually if you want + to run kernel only on systems with h/w unaligned access support in + order to optimise for performance. + +config USE_OF + bool + select OF + select OF_EARLY_FLATTREE + select IRQ_DOMAIN + +config BUILTIN_DTB + bool + +source "kernel/livepatch/Kconfig" + +endmenu + +config ARCH_ENABLE_MEMORY_HOTPLUG + def_bool y + depends on LOONGARCH + +config ARCH_ENABLE_MEMORY_HOTREMOVE + def_bool y + depends on MEMORY_HOTPLUG + +config ARCH_ENABLE_THP_MIGRATION + def_bool y + depends on TRANSPARENT_HUGEPAGE + +config ARCH_MEMORY_PROBE + def_bool y + depends on MEMORY_HOTPLUG + +config GENERIC_BUG + def_bool y + depends on BUG + +config GENERIC_BUG_RELATIVE_POINTERS + def_bool y + depends on GENERIC_BUG + +config LOCKDEP_SUPPORT + bool + default y + +config STACKTRACE_SUPPORT + bool + default y + +config TRACE_IRQFLAGS_SUPPORT + bool + default y + +config MMU + bool + default y + +config ARCH_MMAP_RND_BITS_MIN + default 12 if 64BIT + default 8 + +config ARCH_MMAP_RND_BITS_MAX + default 18 if 64BIT + default 15 + +config ZONE_DMA + bool + +config ZONE_DMA32 + bool + +config KASAN_SHADOW_OFFSET + hex + default 0x0 + depends on KASAN + +menu "Power management options" + +config ARCH_SUSPEND_POSSIBLE + def_bool y + depends on SYS_SUPPORTS_HOTPLUG_CPU || !SMP + +config ARCH_HIBERNATION_POSSIBLE + def_bool y + depends on SYS_SUPPORTS_HOTPLUG_CPU || !SMP + +source "kernel/power/Kconfig" +source "drivers/acpi/Kconfig" +source "drivers/cpufreq/Kconfig" + +endmenu + +source "drivers/firmware/Kconfig" +source "arch/loongarch/kvm/Kconfig" + +config PARAVIRT + bool "Enable paravirtualization code" + help + This changes the kernel so it can modify itself when it is run + under a hypervisor, potentially improving performance significantly + over full virtualization. + +config PARAVIRT_SPINLOCKS + bool "Paravirtualization layer for spinlocks" + depends on PARAVIRT && SMP + help + Paravirtualized spinlocks allow a pvops backend to replace the + spinlock implementation with something virtualization-friendly + (for example, block the virtual CPU rather than spinning). + + It has a minimal impact on native kernels and gives a nice performance + benefit on paravirtualized KVM kernels. + + If you are unsure how to answer this question, answer Y. + +config QUEUED_LOCK_STAT + bool "Paravirt queued spinlock statistics" + depends on PARAVIRT_SPINLOCKS && DEBUG_FS + help + Enable the collection of statistical data on the slowpath + behavior of paravirtualized queued spinlocks and report + them on debugfs. + +config PARAVIRT_TIME_ACCOUNTING + bool "Paravirtual steal time accounting" + select PARAVIRT + default n + help + Select this option to enable fine granularity task steal time + accounting. Time spent executing other tasks in parallel with + the current vCPU is discounted from the vCPU power. To account for + that, there can be a small performance impact. + + If in doubt, say N here. diff --git a/arch/loongarch/Kconfig.debug b/arch/loongarch/Kconfig.debug new file mode 100644 index 0000000000000000000000000000000000000000..94ff96bb511f6d50444b00b67f9a9c2f288a4f6b --- /dev/null +++ b/arch/loongarch/Kconfig.debug @@ -0,0 +1,50 @@ +choice + prompt "Choose kernel unwinder" + default UNWINDER_PROLOGUE if KALLSYMS + help + This determines which method will be used for unwinding kernel stack + traces for panics, oopses, bugs, warnings, perf, /proc//stack, + livepatch, lockdep, and more. + +config UNWINDER_GUESS + bool "Guess unwinder" + help + This option enables the "guess" unwinder for unwinding kernel stack + traces. It scans the stack and reports every kernel text address it + finds. Some of the addresses it reports may be incorrect. + +config UNWINDER_PROLOGUE + bool "Prologue unwinder" + depends on KALLSYMS + help + This option enables the "prologue" unwinder for unwinding kernel stack + traces. It unwind the stack frame based on prologue code analyze. Symbol + information is needed, at least the address and length of each function. + Some of the addresses it reports may be incorrect. + +endchoice + +config DEBUG_ZBOOT + bool "Enable compressed kernel support debugging" + depends on DEBUG_KERNEL && SYS_SUPPORTS_ZBOOT + default n + help + If you want to add compressed kernel support to a new board, and the + board supports uart16550 compatible serial port, please select + SYS_SUPPORTS_ZBOOT_UART16550 for your board and enable this option to + debug it. + + If your board doesn't support uart16550 compatible serial port, you + can try to select SYS_SUPPORTS_ZBOOT and use the other methods to + debug it. for example, add a new serial port support just as + arch/loongarch/boot/compressed/uart-16550.c does. + + After the compressed kernel support works, please disable this option + to reduce the kernel image size and speed up the booting procedure a + little. + +config SPINLOCK_TEST + tristate "Enable spinlock timing tests in debugfs" + default n + help + Add several files to the debugfs to test spinlock speed. diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8c524663f93f5c58850dacdcf971bdfca3eadcca --- /dev/null +++ b/arch/loongarch/Makefile @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Author: Huacai Chen +# Copyright (C) 2020 Loongson Technology Corporation Limited + +archscripts: scripts_basic + $(Q)$(MAKE) $(build)=arch/loongarch/tools elf-entry + $(Q)$(MAKE) $(build)=arch/loongarch/boot/tools relocs + +KBUILD_DEFCONFIG := loongson3_defconfig +KBUILD_DTBS := dtbs + +# +# Select the object file format to substitute into the linker script. +# +64bit-tool-archpref = loongarch64 +32bit-bfd = elf32-loongarch +64bit-bfd = elf64-loongarch +32bit-emul = elf32loongarch +64bit-emul = elf64loongarch + +ifdef CONFIG_64BIT +tool-archpref = $(64bit-tool-archpref) +UTS_MACHINE := loongarch64 +endif + +ifneq ($(SUBARCH),$(ARCH)) + ifeq ($(CROSS_COMPILE),) + CROSS_COMPILE := $(call cc-cross-prefix, $(tool-archpref)-linux- $(tool-archpref)-linux-gnu- $(tool-archpref)-unknown-linux-gnu-) + endif +endif + +ifdef CONFIG_DYNAMIC_FTRACE + KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY + CC_FLAGS_FTRACE := -fpatchable-function-entry=2 +endif + +cflags-y += $(call cc-option, -mno-check-zero-division) + +ifdef CONFIG_64BIT +ld-emul = $(64bit-emul) +cflags-y += -mabi=lp64s +endif + +all-y := vmlinux +all-$(CONFIG_SYS_SUPPORTS_ZBOOT)+= vmlinuz +all-$(CONFIG_EFI_STUB) += vmlinuz.efi + +vmlinuz.efi: vmlinuz + +cflags-y += -pipe -msoft-float +LDFLAGS_vmlinux += -static -n -nostdlib + +# When the assembler supports explicit relocation hint, we must use it. +# GCC may have -mexplicit-relocs off by default if it was built with an old +# assembler, so we force it via an option. +# +# When the assembler does not supports explicit relocation hint, we can't use +# it. Disable it if the compiler supports it. +# +# The combination of a "new" assembler and "old" GCC is not supported, given +# the rarity of this combo and the extra complexity needed to make it work. +# Either upgrade the compiler or downgrade the assembler; the build will error +# out if it is the case (by probing for the model attribute; all supported +# compilers in this case would have support). +# +# Also, -mdirect-extern-access is useful in case of building with explicit +# relocs, for avoiding unnecessary GOT accesses. It is harmless to not have +# support though. +ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS +cflags-y += $(call cc-option,-mexplicit-relocs) +KBUILD_CFLAGS_KERNEL += $(call cc-option,-mdirect-extern-access) +KBUILD_CFLAGS_KERNEL += $(call cc-option,-fdirect-access-external-data) +KBUILD_AFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) +KBUILD_CFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) +else +cflags-y += $(call cc-option,-mno-explicit-relocs) +KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel +KBUILD_CFLAGS_KERNEL += -Wa,-mla-global-with-pcrel +KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs +KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs +endif + +KBUILD_AFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) +KBUILD_CFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) + +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux += --emit-relocs +endif + +cflags-y += -fno-builtin-memcpy -fno-builtin-memmove -fno-builtin-memset + +# +# Kernel compression +# +ifdef CONFIG_SYS_SUPPORTS_ZBOOT +KBUILD_IMAGE = vmlinuz +else +KBUILD_IMAGE = vmlinux +endif + +# +# Board-dependent options and extra files +# +include arch/loongarch/Kbuild.platforms + +entry-y = $(shell $(objtree)/arch/loongarch/tools/elf-entry vmlinux) +drivers-$(CONFIG_PCI) += arch/loongarch/pci/ + +KBUILD_AFLAGS += $(cflags-y) +KBUILD_CFLAGS += $(cflags-y) +KBUILD_CPPFLAGS += -DVMLINUX_LOAD_ADDRESS=$(load-y) + +bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) \ + VMLINUX_ENTRY_ADDRESS=$(entry-y) PLATFORM="$(platform-y)" + +ifdef CONFIG_64BIT +bootvars-y += ADDR_BITS=64 +endif + +# This is required to get dwarf unwinding tables into .debug_frame +# instead of .eh_frame so we don't discard them. +KBUILD_CFLAGS += -fno-asynchronous-unwind-tables +ifdef CONFIG_ARCH_STRICT_ALIGN +KBUILD_CFLAGS += $(call cc-option,-mstrict-align) +else +KBUILD_CFLAGS += $(call cc-option,-mno-strict-align) +endif + +KBUILD_LDFLAGS += -m $(ld-emul) + +ifdef CONFIG_LOONGARCH +CHECKFLAGS += $(shell $(CC) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ + grep -E -vw '__GNUC_(MINOR_|PATCHLEVEL_)?_' | \ + sed -e "s/^\#define /-D'/" -e "s/ /'='/" -e "s/$$/'/" -e 's/\$$/&&/g') +endif + +head-y := arch/loongarch/kernel/head.o + +libs-y += arch/loongarch/lib/ + +# See arch/loongarch/Kbuild for content of core part of the kernel +core-y += arch/loongarch/ + +drivers-y += arch/loongarch/crypto/ + +# suspend and hibernation support +drivers-$(CONFIG_PM) += arch/loongarch/power/ + +ifeq ($(KBUILD_EXTMOD),) +prepare: vdso_prepare +vdso_prepare: prepare0 + $(Q)$(MAKE) $(build)=arch/loongarch/vdso include/generated/vdso-offsets.h +endif + +PHONY += vdso_install +vdso_install: + $(Q)$(MAKE) $(build)=arch/loongarch/vdso $@ + +# boot image targets (arch/loongarch/boot/) +boot-y := vmlinux.bin + +# compressed boot image targets (arch/loongarch/boot/compressed/) +bootz-y := vmlinuz +bootz-y += vmlinuz.bin +bootz-$(CONFIG_EFI_STUB)+= vmlinuz.efi + +all: $(all-y) $(KBUILD_DTBS) + +# boot +$(boot-y): vmlinux FORCE + $(Q)$(MAKE) $(build)=arch/loongarch/boot VMLINUX=vmlinux \ + $(bootvars-y) arch/loongarch/boot/$@ + +ifdef CONFIG_SYS_SUPPORTS_ZBOOT +# boot/compressed +$(bootz-y): vmlinux FORCE + $(Q)$(MAKE) $(build)=arch/loongarch/boot/compressed \ + $(bootvars-y) 64bit-bfd=$(64bit-bfd) $@ +else +vmlinuz: FORCE + @echo ' CONFIG_SYS_SUPPORTS_ZBOOT is not enabled' + /bin/false +endif + +CLEAN_FILES += vmlinux + +install: + $(Q)install -D -m 755 vmlinux $(INSTALL_PATH)/vmlinux-$(KERNELRELEASE) +ifdef CONFIG_SYS_SUPPORTS_ZBOOT + $(Q)install -D -m 755 vmlinuz $(INSTALL_PATH)/vmlinuz-$(KERNELRELEASE) +endif + $(Q)install -D -m 644 .config $(INSTALL_PATH)/config-$(KERNELRELEASE) + $(Q)install -D -m 644 System.map $(INSTALL_PATH)/System.map-$(KERNELRELEASE) + +archclean: + $(Q)$(MAKE) $(clean)=arch/loongarch/boot + $(Q)$(MAKE) $(clean)=arch/loongarch/boot/compressed + $(Q)$(MAKE) $(clean)=arch/loongarch/boot/tools + $(Q)$(MAKE) $(clean)=arch/loongarch/tools + +define archhelp + echo ' install - install kernel into $(INSTALL_PATH)' + echo ' vmlinux.bin - Raw binary boot image' + echo ' vmlinuz - Compressed boot(zboot) image' + echo ' vmlinuz.bin - Raw binary zboot image' + echo +endef diff --git a/arch/loongarch/Makefile.postlink b/arch/loongarch/Makefile.postlink new file mode 100644 index 0000000000000000000000000000000000000000..46aa6050a7d147fb42734d3f65fd74c94252c269 --- /dev/null +++ b/arch/loongarch/Makefile.postlink @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0 +# =========================================================================== +# Post-link LoongArch pass +# =========================================================================== +# +# 1. Insert relocations into vmlinux + +PHONY := __archpost +__archpost: + +-include include/config/auto.conf +include scripts/Kbuild.include + +CMD_RELOCS = arch/loongarch/boot/tools/relocs --keep +quiet_cmd_relocs = RELOCS $@ + cmd_relocs = $(CMD_RELOCS) $@ + +quiet_cmd_strip_relocs = RSTRIP $@ + cmd_strip_relocs = \ + $(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' \ + --remove-section='.rela.*' --remove-section='.rela__*' $@ + +# `@true` prevents complaint when there is nothing to be done + +vmlinux: FORCE + @true +ifeq ($(CONFIG_RELOCATABLE),y) + $(call cmd,relocs) + $(call cmd,strip_relocs) +endif + +%.ko: FORCE + @true + +clean: + @true + +PHONY += FORCE clean + +FORCE: + +.PHONY: $(PHONY) diff --git a/arch/loongarch/boot/Makefile b/arch/loongarch/boot/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..91ef912e20768da690dc62bb6ed89d5b9d7c47a3 --- /dev/null +++ b/arch/loongarch/boot/Makefile @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Author: Huacai Chen +# Copyright (C) 2020 Loongson Technology Corporation Limited + +# +# Drop some uninteresting sections in the kernel. +# This is only relevant for ELF kernels but doesn't hurt a.out +# +OBJECT_FILES_NON_STANDARD := y + +drop-sections := .comment .note .options +strip-flags := $(addprefix --remove-section=,$(drop-sections)) + +suffix-y := bin +suffix-$(CONFIG_KERNEL_GZIP) := gz +suffix-$(CONFIG_KERNEL_BZIP2) := bz2 +suffix-$(CONFIG_KERNEL_LZMA) := lzma +suffix-$(CONFIG_KERNEL_LZO) := lzo +suffix-$(CONFIG_KERNEL_LZ4) := lz4 +suffix-$(CONFIG_KERNEL_ZSTD) := zst + +targets += vmlinux.bin +quiet_cmd_bin = OBJCOPY $@ + cmd_bin = $(OBJCOPY) -O binary $(strip-flags) $(VMLINUX) $@ +$(obj)/vmlinux.bin: $(VMLINUX) FORCE + $(call if_changed,bin) + +# +# Compressed vmlinux images +# + +extra-y += vmlinux.bin.gz +extra-y += vmlinux.bin.bz2 +extra-y += vmlinux.bin.lzma +extra-y += vmlinux.bin.lzo +extra-y += vmlinux.bin.lz4 +extra-y += vmlinux.bin.zst + +$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin FORCE + $(call if_changed,bzip2) + +$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE + $(call if_changed,gzip) + +$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzma) + +$(obj)/vmlinux.bin.lzo: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzo) + +$(obj)/vmlinux.bin.lz4: $(obj)/vmlinux.bin FORCE + $(call if_changed,lz4) + +$(obj)/vmlinux.bin.zst: $(obj)/vmlinux.bin FORCE + $(call if_changed,zst22) diff --git a/arch/loongarch/boot/compressed/Makefile b/arch/loongarch/boot/compressed/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..60259def78ec839f87b8f5057dbb72379a74130c --- /dev/null +++ b/arch/loongarch/boot/compressed/Makefile @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Author: Huacai Chen +# Copyright (C) 2020 Loongson Technology Corporation Limited + +OBJECT_FILES_NON_STANDARD := y + +include $(srctree)/arch/loongarch/Kbuild.platforms + +# set the default size of the mallocing area for decompressing +BOOT_HEAP_SIZE := 0x400000 + +ifdef CONFIG_KASAN +KASAN_SANITIZE := n +endif + +# Disable Function Tracer +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE), $(KBUILD_CFLAGS)) + +KBUILD_CFLAGS := $(filter-out -fstack-protector, $(KBUILD_CFLAGS)) + +KBUILD_CFLAGS += -fpie -include $(srctree)/include/linux/hidden.h + +KBUILD_CFLAGS := $(KBUILD_CFLAGS) -D__KERNEL__ \ + -DBOOT_HEAP_SIZE=$(BOOT_HEAP_SIZE) \ + -D"VMLINUX_LOAD_ADDRESS_ULL=$(VMLINUX_LOAD_ADDRESS)ull" \ + -D"KERNEL_ENTRY=$(VMLINUX_ENTRY_ADDRESS)ull" \ + +KBUILD_AFLAGS := $(KBUILD_AFLAGS) -D__ASSEMBLY__ \ + -DBOOT_HEAP_SIZE=$(BOOT_HEAP_SIZE) \ + -DKERNEL_ENTRY=$(VMLINUX_ENTRY_ADDRESS) + +# decompressor objects (linked with vmlinuz) +vmlinuzobjs-y := $(obj)/head.o $(obj)/decompress.o $(obj)/string.o + +ifdef CONFIG_DEBUG_ZBOOT +vmlinuzobjs-$(CONFIG_DEBUG_ZBOOT) += $(obj)/dbg.o +vmlinuzobjs-$(CONFIG_DEBUG_ZBOOT) += $(obj)/uart-16550.o +endif + +vmlinuzobjs-$(CONFIG_KERNEL_XZ) += $(obj)/ashldi3.o + +extra-y += ashldi3.c +$(obj)/ashldi3.c: $(obj)/%.c: $(srctree)/lib/%.c FORCE + $(call if_changed,shipped) + +targets := $(notdir $(vmlinuzobjs-y)) + +targets += vmlinux.bin +OBJCOPYFLAGS_vmlinux.bin := $(OBJCOPYFLAGS) -O binary -R .comment -S +$(obj)/vmlinux.bin: vmlinux FORCE + $(call if_changed,objcopy) + +tool_$(CONFIG_KERNEL_GZIP) = gzip +tool_$(CONFIG_KERNEL_BZIP2) = bzip2 +tool_$(CONFIG_KERNEL_LZ4) = lz4 +tool_$(CONFIG_KERNEL_LZMA) = lzma +tool_$(CONFIG_KERNEL_LZO) = lzo +tool_$(CONFIG_KERNEL_XZ) = xzkern +tool_$(CONFIG_KERNEL_ZSTD) = zstd22 + +targets += vmlinux.bin.z +$(obj)/vmlinux.bin.z: $(obj)/vmlinux.bin FORCE + $(call if_changed,$(tool_y)) + +targets += piggy.o dummy.o +OBJCOPYFLAGS_piggy.o := --add-section=.image=$(obj)/vmlinux.bin.z \ + --set-section-flags=.image=contents,alloc,load,readonly,data +$(obj)/piggy.o: $(obj)/dummy.o $(obj)/vmlinux.bin.z FORCE + $(call if_changed,objcopy) + +HOSTCFLAGS_calc_vmlinuz_load_addr.o += $(LINUXINCLUDE) + +# Calculate the load address of the compressed kernel image +hostprogs := calc_vmlinuz_load_addr + +ifneq ($(zload-y),) +VMLINUZ_LOAD_ADDRESS := $(zload-y) +else +VMLINUZ_LOAD_ADDRESS = $(shell $(obj)/calc_vmlinuz_load_addr \ + $(obj)/vmlinux.bin $(VMLINUX_LOAD_ADDRESS)) +endif +UIMAGE_LOADADDR = $(VMLINUZ_LOAD_ADDRESS) + +vmlinuzobjs-y += $(obj)/piggy.o + +quiet_cmd_zld = LD $@ + cmd_zld = $(LD) $(KBUILD_LDFLAGS) -Ttext $(VMLINUZ_LOAD_ADDRESS) -T $< $(vmlinuzobjs-y) -o $@ +quiet_cmd_strip = STRIP $@ + cmd_strip = $(STRIP) -s $@ +vmlinuz: $(src)/ld.script $(vmlinuzobjs-y) $(obj)/calc_vmlinuz_load_addr + $(call cmd,zld) + $(call cmd,strip) + +efiobjs-y := $(obj)/efi-header.o $(obj)/decompress.o $(obj)/piggy.o \ + $(obj)/string.o $(objtree)/drivers/firmware/efi/libstub/lib.a + +efiobjs-$(CONFIG_DEBUG_ZBOOT) += $(obj)/dbg.o +efiobjs-$(CONFIG_DEBUG_ZBOOT) += $(obj)/uart-16550.o + +targets += $(notdir $(efiobjs-y)) + +quiet_cmd_efild = LD $@ + cmd_efild = $(LD) $(KBUILD_LDFLAGS) -T $< $(efiobjs-y) -o $@ + +quiet_cmd_eficopy = OBJCOPY $@ +cmd_eficopy = $(OBJCOPY) $(OBJCOPYFLAGS) -O binary -R .comment -S $@ $@ + +vmlinuz.efi: $(src)/ld.script $(efiobjs-y) + $(call cmd,efild) + $(call cmd,eficopy) + +clean-files += $(objtree)/vmlinuz +clean-files += $(objtree)/vmlinuz.bin +clean-files += $(objtree)/vmlinuz.efi diff --git a/arch/loongarch/boot/compressed/calc_vmlinuz_load_addr.c b/arch/loongarch/boot/compressed/calc_vmlinuz_load_addr.c new file mode 100644 index 0000000000000000000000000000000000000000..d14f75ec827323702d61d6ef6eb2871ecfda3513 --- /dev/null +++ b/arch/loongarch/boot/compressed/calc_vmlinuz_load_addr.c @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2010 "Wu Zhangjin" + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + unsigned long long vmlinux_size, vmlinux_load_addr, vmlinuz_load_addr; + struct stat sb; + + if (argc != 3) { + fprintf(stderr, "Usage: %s \n", + argv[0]); + return EXIT_FAILURE; + } + + if (stat(argv[1], &sb) == -1) { + perror("stat"); + return EXIT_FAILURE; + } + + /* Convert hex characters to dec number */ + errno = 0; + if (sscanf(argv[2], "%llx", &vmlinux_load_addr) != 1) { + if (errno != 0) + perror("sscanf"); + else + fprintf(stderr, "No matching characters\n"); + + return EXIT_FAILURE; + } + + vmlinux_size = (uint64_t)sb.st_size; + vmlinuz_load_addr = vmlinux_load_addr + vmlinux_size; + + /* + * Align with 64KB: KEXEC needs load sections to be aligned to PAGE_SIZE, + * which may be as large as 64KB depending on the kernel configuration. + */ + + vmlinuz_load_addr += (SZ_64K - vmlinux_size % SZ_64K); + + printf("0x%llx\n", vmlinuz_load_addr); + + return EXIT_SUCCESS; +} diff --git a/arch/loongarch/boot/compressed/dbg.c b/arch/loongarch/boot/compressed/dbg.c new file mode 100644 index 0000000000000000000000000000000000000000..329193f0b0596785a2d7da6a5ba0b1e73db7b029 --- /dev/null +++ b/arch/loongarch/boot/compressed/dbg.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LoongArch-specific debug support for pre-boot environment + * + * NOTE: putc() is board specific, if your board have a 16550 compatible uart, + * please select SYS_SUPPORTS_ZBOOT_UART16550 for your machine. othewise, you + * need to implement your own putc(). + */ +#include +#include + +void __weak putc(char c) +{ +} + +void puts(const char *s) +{ + char c; + while ((c = *s++) != '\0') { + putc(c); + if (c == '\n') + putc('\r'); + } +} + +void puthex(unsigned long long val) +{ + + unsigned char buf[10]; + int i; + for (i = 7; i >= 0; i--) { + buf[i] = "0123456789ABCDEF"[val & 0x0F]; + val >>= 4; + } + buf[8] = '\0'; + puts(buf); +} diff --git a/arch/loongarch/boot/compressed/decompress.c b/arch/loongarch/boot/compressed/decompress.c new file mode 100644 index 0000000000000000000000000000000000000000..b5fadf88628eeea940e1efe52120a045f9391d18 --- /dev/null +++ b/arch/loongarch/boot/compressed/decompress.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#include +#include +#include +#include + +#include + +/* + * These two variables specify the free mem region + * that can be used for temporary malloc area + */ +unsigned long free_mem_ptr; +unsigned long free_mem_end_ptr; + +/* The linker tells us where the image is. */ +extern unsigned char __image_begin, __image_end; + +/* debug interfaces */ +#ifdef CONFIG_DEBUG_ZBOOT +extern void puts(const char *s); +extern void puthex(unsigned long long val); +#else +#define puts(s) do {} while (0) +#define puthex(val) do {} while (0) +#endif + +void error(char *x) +{ + puts("\n\n"); + puts(x); + puts("\n\n -- System halted"); + + while (1) + ; /* Halt */ +} + +/* activate the code for pre-boot environment */ +#define STATIC static + +#ifdef CONFIG_KERNEL_GZIP +#include "../../../../lib/decompress_inflate.c" +#endif + +#ifdef CONFIG_KERNEL_BZIP2 +#include "../../../../lib/decompress_bunzip2.c" +#endif + +#ifdef CONFIG_KERNEL_LZ4 +#include "../../../../lib/decompress_unlz4.c" +#endif + +#ifdef CONFIG_KERNEL_LZMA +#include "../../../../lib/decompress_unlzma.c" +#endif + +#ifdef CONFIG_KERNEL_LZO +#include "../../../../lib/decompress_unlzo.c" +#endif + +#ifdef CONFIG_KERNEL_XZ +#include "../../../../lib/decompress_unxz.c" +#endif + +#ifdef CONFIG_KERNEL_ZSTD +#include "../../../../lib/decompress_unzstd.c" +#endif + +const unsigned long __stack_chk_guard = 0x000a0dff; + +void __stack_chk_fail(void) +{ + error("stack-protector: Kernel stack is corrupted\n"); +} + +void decompress_kernel(unsigned long boot_heap_start, long kdump_reloc_offset) +{ + unsigned long zimage_start, zimage_size; + + zimage_start = (unsigned long)(&__image_begin); + zimage_size = (unsigned long)(&__image_end) - + (unsigned long)(&__image_begin); + + puts("zimage at: "); + puthex(zimage_start); + puts(" "); + puthex(zimage_size + zimage_start); + puts("\n"); + + /* This area are prepared for mallocing when decompressing */ + free_mem_ptr = boot_heap_start; + free_mem_end_ptr = boot_heap_start + BOOT_HEAP_SIZE; + + /* Display standard Linux/LoongArch boot prompt */ + puts("Uncompressing Linux at load address "); + puthex(VMLINUX_LOAD_ADDRESS_ULL); + puts("\n"); + + /* Decompress the kernel with according algorithm */ + __decompress((char *)zimage_start, zimage_size, 0, 0, + (void *)VMLINUX_LOAD_ADDRESS_ULL + kdump_reloc_offset, 0, 0, error); + + puts("Now, booting the kernel...\n"); +} diff --git a/arch/loongarch/boot/compressed/dummy.c b/arch/loongarch/boot/compressed/dummy.c new file mode 100644 index 0000000000000000000000000000000000000000..31dbf45bf99c5996ca0ee62cba186d912b08e666 --- /dev/null +++ b/arch/loongarch/boot/compressed/dummy.c @@ -0,0 +1,4 @@ +int main(void) +{ + return 0; +} diff --git a/arch/loongarch/boot/compressed/efi-header.S b/arch/loongarch/boot/compressed/efi-header.S new file mode 100644 index 0000000000000000000000000000000000000000..f2e536cdbf720a6fca175675aec800b35d64fdf7 --- /dev/null +++ b/arch/loongarch/boot/compressed/efi-header.S @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2013 - 2017 Linaro, Ltd. + * Copyright (C) 2013, 2014 Red Hat, Inc. + * Copyright (C) 2020, 2021 Loongson, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include + + .macro __EFI_PE_HEADER + .long PE_MAGIC +coff_header: + .short IMAGE_FILE_MACHINE_LOONGARCH /* Machine */ + .short section_count /* NumberOfSections */ + .long 0 /* TimeDateStamp */ + .long 0 /* PointerToSymbolTable */ + .long 0 /* NumberOfSymbols */ + .short section_table - optional_header /* SizeOfOptionalHeader */ + .short IMAGE_FILE_DEBUG_STRIPPED | \ + IMAGE_FILE_EXECUTABLE_IMAGE | \ + IMAGE_FILE_LINE_NUMS_STRIPPED /* Characteristics */ + +optional_header: + .short PE_OPT_MAGIC_PE32PLUS /* PE32+ format */ + .byte 0x02 /* MajorLinkerVersion */ + .byte 0x14 /* MinorLinkerVersion */ + .long _data - efi_header_end /* SizeOfCode */ + .long _end - _data /* SizeOfInitializedData */ + .long 0 /* SizeOfUninitializedData */ + .long efi_pe_entry - _text /* AddressOfEntryPoint */ + .long efi_header_end - _text /* BaseOfCode */ + +extra_header_fields: + .quad 0 /* ImageBase */ + .long PECOFF_SEGMENT_ALIGN /* SectionAlignment */ + .long PECOFF_FILE_ALIGN /* FileAlignment */ + .short 0 /* MajorOperatingSystemVersion */ + .short 0 /* MinorOperatingSystemVersion */ + .short LINUX_EFISTUB_MAJOR_VERSION /* MajorImageVersion */ + .short LINUX_EFISTUB_MINOR_VERSION /* MinorImageVersion */ + .short 0 /* MajorSubsystemVersion */ + .short 0 /* MinorSubsystemVersion */ + .long 0 /* Win32VersionValue */ + + .long _end - _text /* SizeOfImage */ + + /* Everything before the kernel image is considered part of the header */ + .long efi_header_end - _head /* SizeOfHeaders */ + .long 0 /* CheckSum */ + .short IMAGE_SUBSYSTEM_EFI_APPLICATION /* Subsystem */ + .short 0 /* DllCharacteristics */ + .quad 0 /* SizeOfStackReserve */ + .quad 0 /* SizeOfStackCommit */ + .quad 0 /* SizeOfHeapReserve */ + .quad 0 /* SizeOfHeapCommit */ + .long 0 /* LoaderFlags */ + .long (section_table - .) / 8 /* NumberOfRvaAndSizes */ + + .quad 0 /* ExportTable */ + .quad 0 /* ImportTable */ + .quad 0 /* ResourceTable */ + .quad 0 /* ExceptionTable */ + .quad 0 /* CertificationTable */ + .quad 0 /* BaseRelocationTable */ + + /* Section table */ +section_table: + .ascii ".text\0\0\0" + .long _data - efi_header_end /* VirtualSize */ + .long efi_header_end - _text /* VirtualAddress */ + .long _data - efi_header_end /* SizeOfRawData */ + .long efi_header_end - _text /* PointerToRawData */ + + .long 0 /* PointerToRelocations */ + .long 0 /* PointerToLineNumbers */ + .short 0 /* NumberOfRelocations */ + .short 0 /* NumberOfLineNumbers */ + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE /* Characteristics */ + + .ascii ".data\0\0\0" + .long _end - _data /* VirtualSize */ + .long _data - _text /* VirtualAddress */ + .long _edata - _data /* SizeOfRawData */ + .long _data - _text /* PointerToRawData */ + + .long 0 /* PointerToRelocations */ + .long 0 /* PointerToLineNumbers */ + .short 0 /* NumberOfRelocations */ + .short 0 /* NumberOfLineNumbers */ + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_WRITE /* Characteristics */ + + .org 0x20e + .word kernel_version - 512 - _head + + .set section_count, (. - section_table) / 40 + + .balign 0x10000 /* PECOFF_SEGMENT_ALIGN */ +efi_header_end: + .endm + + .section ".head.text","ax" +_head: + /* "MZ", MS-DOS header */ + .byte 0x4d + .byte 0x5a + .org 0x3c + /* Offset to the PE header */ + .long pe_header - _head + +pe_header: + __EFI_PE_HEADER + +start: + .globl start + +kernel_entaddr: + .quad KERNEL_ENTRY + .globl kernel_entaddr + +kernel_version: + .ascii UTS_RELEASE " (" LINUX_COMPILE_BY "@" LINUX_COMPILE_HOST ") " UTS_VERSION "\0" diff --git a/arch/loongarch/boot/compressed/head.S b/arch/loongarch/boot/compressed/head.S new file mode 100644 index 0000000000000000000000000000000000000000..b2f676c5b4fc167101102082dac6443be48b7d3e --- /dev/null +++ b/arch/loongarch/boot/compressed/head.S @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#include +#include +#include +#include +#include + +SYM_CODE_START(start) + /* Save boot rom start args */ + move s0, a0 + move s1, a1 + move s2, a2 + move s3, a3 /* for kdump */ + + /* Config Direct Mapping */ + li.d t0, CSR_DMW0_INIT + csrwr t0, LOONGARCH_CSR_DMWIN0 + li.d t0, CSR_DMW1_INIT + csrwr t0, LOONGARCH_CSR_DMWIN1 + + /* Clear BSS */ + la.pcrel a0, _edata + la.pcrel a2, _end +1: st.d zero, a0, 0 + addi.d a0, a0, 8 + bne a2, a0, 1b + + la.pcrel a0, .heap /* heap address */ + la.pcrel sp, .stack + 8192 /* stack address */ + + la.pcrel a0, .heap /* heap address */ + move a1, a3 /* kdump relocate offset */ + bnez a3, 2f + li.w a1, 0 + li.w s3, 0 +2: + la.pcrel ra, 3f + la.pcrel t4, decompress_kernel + jirl zero, t4, 0 +3: + move a0, s0 + move a1, s1 + move a2, s2 + move a3, s3 + PTR_LI t4, KERNEL_ENTRY + add.d t4, t4, a3 + jirl zero, t4, 0 +4: + b 4b +SYM_CODE_END(start) + + .comm .heap,BOOT_HEAP_SIZE,4 + .comm .stack,4096*2,4 diff --git a/arch/loongarch/boot/compressed/ld.script b/arch/loongarch/boot/compressed/ld.script new file mode 100644 index 0000000000000000000000000000000000000000..671d2027c7b02aaa44e7f3750d268388896fd15d --- /dev/null +++ b/arch/loongarch/boot/compressed/ld.script @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ld.script for compressed kernel support of LoongArch + * + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +/* + * Max avaliable Page Size is 64K, so we set SectionAlignment + * field of EFI application to 64K. + */ +PECOFF_FILE_ALIGN = 0x200; +PECOFF_SEGMENT_ALIGN = 0x10000; + +OUTPUT_ARCH(loongarch) +ENTRY(start) +PHDRS { + text PT_LOAD FLAGS(7); /* RWX */ +} +SECTIONS +{ + /* Text and read-only data */ + _text = .; + .head.text : { + *(.head.text) + } + + .text : { + *(.text) + *(.init.text) + *(.rodata) + }: text + /* End of text section */ + + . = ALIGN(PECOFF_SEGMENT_ALIGN); + _data = .; + /* Writable data */ + .data : { + *(.data) + *(.init.data) + /* Put the compressed image here */ + __image_begin = .; + *(.image) + __image_end = .; + CONSTRUCTORS + . = ALIGN(PECOFF_FILE_ALIGN); + } + + _edata = .; + /* End of data section */ + + /* BSS */ + .bss : { + *(.bss) + *(.init.bss) + } + . = ALIGN(PECOFF_SEGMENT_ALIGN); + _end = .; + + /* Sections to be discarded */ + /DISCARD/ : { + *(.options) + *(.comment) + *(.note) + } +} diff --git a/arch/loongarch/boot/compressed/string.c b/arch/loongarch/boot/compressed/string.c new file mode 100644 index 0000000000000000000000000000000000000000..50bf9c2f841c9846bc7d23ca2b031076035f281f --- /dev/null +++ b/arch/loongarch/boot/compressed/string.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/loongarch/boot/compressed/string.c + * + * Very small subset of simple string routines + */ + +#include + +void * __weak memchr(const void *s, int c, size_t n) +{ + const unsigned char *p = s; + while (n-- != 0) { + if ((unsigned char)c == *p++) { + return (void *)(p - 1); + } + } + return NULL; +} + +int __weak memcmp(const void *cs, const void *ct, size_t count) +{ + int res = 0; + const unsigned char *su1, *su2; + + for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) { + res = *su1 - *su2; + if (res != 0) + break; + } + return res; +} + +void __weak *memset(void *s, int c, size_t n) +{ + int i; + char *ss = s; + + for (i = 0; i < n; i++) + ss[i] = c; + return s; +} + +void __weak *memcpy(void *dest, const void *src, size_t n) +{ + int i; + const char *s = src; + char *d = dest; + + for (i = 0; i < n; i++) + d[i] = s[i]; + return dest; +} + +void __weak *memmove(void *dest, const void *src, size_t n) +{ + int i; + const char *s = src; + char *d = dest; + + if (d < s) { + for (i = 0; i < n; i++) + d[i] = s[i]; + } else if (d > s) { + for (i = n - 1; i >= 0; i--) + d[i] = s[i]; + } + + return dest; +} + +int __weak strcmp(const char *str1, const char *str2) +{ + int delta = 0; + const unsigned char *s1 = (const unsigned char *)str1; + const unsigned char *s2 = (const unsigned char *)str2; + + while (*s1 || *s2) { + delta = *s1 - *s2; + if (delta) + return delta; + s1++; + s2++; + } + return 0; +} + +size_t __weak strlen(const char *s) +{ + const char *sc; + + for (sc = s; *sc != '\0'; ++sc) + /* nothing */; + return sc - s; +} + +size_t __weak strnlen(const char *s, size_t count) +{ + const char *sc; + + for (sc = s; count-- && *sc != '\0'; ++sc) + /* nothing */; + return sc - s; +} + +char * __weak strrchr(const char *s, int c) +{ + const char *last = NULL; + do { + if (*s == (char)c) + last = s; + } while (*s++); + return (char *)last; +} + +#ifdef CONFIG_KASAN +extern void *__memset(void *s, int c, size_t n) __alias(memset); +extern void *__memcpy(void *dest, const void *src, size_t n) __alias(memcpy); +extern void *__memmove(void *dest, const void *src, size_t n) __alias(memmove); +#endif diff --git a/arch/loongarch/boot/compressed/uart-16550.c b/arch/loongarch/boot/compressed/uart-16550.c new file mode 100644 index 0000000000000000000000000000000000000000..32051e8c717dc76ee56d6d883954850836a4caee --- /dev/null +++ b/arch/loongarch/boot/compressed/uart-16550.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * 16550 compatible uart based serial debug support for zboot + */ + +#include +#include + +#include + +#define UART_BASE 0x1fe001e0 +#define PORT(offset) (TO_UNCACHE(UART_BASE) + (offset)) + +#ifndef IOTYPE +#define IOTYPE char +#endif + +#ifndef PORT +#error please define the serial port address for your own machine +#endif + +static inline unsigned int serial_in(int offset) +{ + return *((volatile IOTYPE *)PORT(offset)) & 0xFF; +} + +static inline void serial_out(int offset, int value) +{ + *((volatile IOTYPE *)PORT(offset)) = value & 0xFF; +} + +void putc(char c) +{ + int timeout = 1000000; + + while (((serial_in(UART_LSR) & UART_LSR_THRE) == 0) && (timeout-- > 0)) + ; + + serial_out(UART_TX, c); +} diff --git a/arch/loongarch/boot/dts/Makefile b/arch/loongarch/boot/dts/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..94c45818e4c9edafc645220fa7b8a0dd632c5b5d --- /dev/null +++ b/arch/loongarch/boot/dts/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +subdir-y += loongson + +obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) diff --git a/arch/loongarch/boot/dts/loongson/Makefile b/arch/loongarch/boot/dts/loongson/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9def5f6ee9c6e76b05ee248de71f4ad0b22142c0 --- /dev/null +++ b/arch/loongarch/boot/dts/loongson/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_CPU_LOONGSON64) += loongson3_ls7a.dtb + +obj-y += $(patsubst %.dtb, %.dtb.o, $(dtb-y)) diff --git a/arch/loongarch/boot/dts/loongson/loongson3.dtsi b/arch/loongarch/boot/dts/loongson/loongson3.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..2f985a7ddaf21994d554aae6e563487c8ecf3d43 --- /dev/null +++ b/arch/loongarch/boot/dts/loongson/loongson3.dtsi @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/ { + /* + * Loongson-3 may have as many as 4 nodes, each node has 4 cores. + * Each core has its own pcache and cores in the same node share scache. + */ + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x0>; + l2-cache = <&vcache0>; + next-level-cache = <&scache0>; + }; + + cpu@1 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x1>; + l2-cache = <&vcache1>; + next-level-cache = <&scache0>; + }; + + cpu@2 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x2>; + l2-cache = <&vcache2>; + next-level-cache = <&scache0>; + }; + + cpu@3 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x3>; + l2-cache = <&vcache3>; + next-level-cache = <&scache0>; + }; + + cpu@4 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x4>; + l2-cache = <&vcache4>; + next-level-cache = <&scache1>; + }; + + cpu@5 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x5>; + l2-cache = <&vcache5>; + next-level-cache = <&scache1>; + }; + + cpu@6 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x6>; + l2-cache = <&vcache6>; + next-level-cache = <&scache1>; + }; + + cpu@7 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x7>; + l2-cache = <&vcache7>; + next-level-cache = <&scache1>; + }; + + cpu@8 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x8>; + l2-cache = <&vcache8>; + next-level-cache = <&scache2>; + }; + + cpu@9 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x9>; + l2-cache = <&vcache9>; + next-level-cache = <&scache2>; + }; + + cpu@a { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xa>; + l2-cache = <&vcachea>; + next-level-cache = <&scache2>; + }; + + cpu@b { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xb>; + l2-cache = <&vcacheb>; + next-level-cache = <&scache2>; + }; + + cpu@c { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xc>; + l2-cache = <&vcachec>; + next-level-cache = <&scache3>; + }; + + cpu@d { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xd>; + l2-cache = <&vcached>; + next-level-cache = <&scache3>; + }; + + cpu@e { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xe>; + l2-cache = <&vcachee>; + next-level-cache = <&scache3>; + }; + + cpu@f { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0xf>; + l2-cache = <&vcachef>; + next-level-cache = <&scache3>; + }; + + vcache0: l2-cache0 { + compatible = "cache"; + next-level-cache = <&scache0>; + }; + + vcache1: l2-cache1 { + compatible = "cache"; + next-level-cache = <&scache0>; + }; + + vcache2: l2-cache2 { + compatible = "cache"; + next-level-cache = <&scache0>; + }; + + vcache3: l2-cache3 { + compatible = "cache"; + next-level-cache = <&scache0>; + }; + + vcache4: l2-cache4 { + compatible = "cache"; + next-level-cache = <&scache1>; + }; + + vcache5: l2-cache5 { + compatible = "cache"; + next-level-cache = <&scache1>; + }; + + vcache6: l2-cache6 { + compatible = "cahce"; + next-level-cache = <&scache1>; + }; + + vcache7: l2-cache7 { + compatible = "cache"; + next-level-cache = <&scache1>; + }; + + vcache8: l2-cache8 { + compatible = "cache"; + next-level-cache = <&scache2>; + }; + + vcache9: l2-cache9 { + compatible = "cache"; + next-level-cache = <&scache2>; + }; + + vcachea: l2-cachea { + compatible = "cache"; + next-level-cache = <&scache2>; + }; + + vcacheb: l2-cacheb { + compatible = "cache"; + next-level-cache = <&scache2>; + }; + + vcachec: l2-cachec { + compatible = "cache"; + next-level-cache = <&scache3>; + }; + + vcached: l2-cached { + compatible = "cache"; + next-level-cache = <&scache3>; + }; + + vcachee: l2-cachee { + compatible = "cache"; + next-level-cache = <&scache3>; + }; + + vcachef: l2-cachef { + compatible = "cache"; + next-level-cache = <&scache3>; + }; + + scache0: l3-cache0 { + compatible = "cache"; + }; + + scache1: l3-cache1 { + compatible = "cache"; + }; + + scache2: l3-cache2 { + compatible = "cache"; + }; + + scache3: l3-cache3 { + compatible = "cache"; + }; + }; +}; diff --git a/arch/loongarch/boot/dts/loongson/loongson3_ls7a.dts b/arch/loongarch/boot/dts/loongson/loongson3_ls7a.dts new file mode 100644 index 0000000000000000000000000000000000000000..f9fa2d5d20326a29422d5a0b91f09ab65ae03448 --- /dev/null +++ b/arch/loongarch/boot/dts/loongson/loongson3_ls7a.dts @@ -0,0 +1,159 @@ +/dts-v1/; +#include "loongson3.dtsi" +/ { + model = "loongson,generic"; + compatible = "loongson,loongson3"; + #address-cells = <2>; + #size-cells = <2>; + + memory { + name = "memory"; + device_type = "memory"; + }; + + cpuic: interrupt-controller { + compatible = "loongson,cpu-interrupt-controller"; + interrupt-controller; + #interrupt-cells = <1>; + }; + + platic: interrupt-controller@1bd00040 { + compatible = "loongson,ls7a-interrupt-controller"; + interrupt-controller; + #interrupt-cells = <1>; + interrupts = <3>; + interrupt-parent = <&cpuic>; + }; + + aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + }; + + platform { + compatible = "loongson,nbus", "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + enable-lpc-irq; + ranges = <0x000 0x00000000 0x000 0x00000000 0x20000000 + 0x000 0x40000000 0x000 0x40000000 0x40000000 + 0xe00 0x00000000 0xe00 0x00000000 0x80000000>; + + uart0: serial@10080000 { + device_type = "serial"; + compatible = "ns16550,loongson"; + reg = <0 0x10080000 0x100>; + clock-frequency = <50000000>; + interrupts = <72>; + interrupt-parent = <&platic>; + no-loopback-test; + }; + + gpio: gpio@100e0000 { + compatible = "loongson,ls7a-gpio"; + reg = <0 0x100e0000 0xc00>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <57>; + conf_offset = <0x800>; + out_offset = <0x900>; + in_offset = <0xa00>; + gpio_base = <16>; + interrupts = <124>; + interrupt-parent = <&platic>; + }; + + i2c0: i2c@10090000 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090000 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c1: i2c@10090100 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090100 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c2: i2c@10090200 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090200 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c3: i2c@10090300 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090300 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c4: i2c@10090400 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090400 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c5: i2c@10090500 { + compatible = "loongson,ls7a-i2c"; + reg = <0 0x10090500 0x8>; + interrupts = <73>; + interrupt-parent = <&platic>; + #address-cells = <1>; + #size-cells = <0>; + }; + + rtc0: rtc@100d0100 { + compatible = "loongson,ls7a-rtc"; + reg = <0 0x100d0100 0x100>; + interrupts = <116>; + interrupt-parent = <&platic>; + }; + + pwm0: pwm@100a0000 { + compatible = "loongson,ls7a-pwm"; + reg = <0 0x100a0000 0x10>; + interrupts = <88>; + interrupt-parent = <&platic>; + }; + + pwm1: pwm@100a0100 { + compatible = "loongson,ls7a-pwm"; + reg = <0 0x100a0100 0x10>; + interrupts = <89>; + interrupt-parent = <&platic>; + }; + + pwm2: pwm@100a0200 { + compatible = "loongson,ls7a-pwm"; + reg = <0 0x100a0200 0x10>; + interrupts = <90>; + interrupt-parent = <&platic>; + }; + + pwm3: pwm@100a0300 { + compatible = "loongson,ls7a-pwm"; + reg = <0 0x100a0300 0x10>; + interrupts = <91>; + interrupt-parent = <&platic>; + }; + }; +}; diff --git a/arch/loongarch/boot/tools/Makefile b/arch/loongarch/boot/tools/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..592e05a51a4ae66187e54d8da239df876c9cde14 --- /dev/null +++ b/arch/loongarch/boot/tools/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 + +hostprogs += relocs +relocs-objs += relocs_32.o +relocs-objs += relocs_64.o +relocs-objs += relocs_main.o +PHONY += relocs +relocs: $(obj)/relocs + @: diff --git a/arch/loongarch/boot/tools/relocs.c b/arch/loongarch/boot/tools/relocs.c new file mode 100644 index 0000000000000000000000000000000000000000..e5199c33d304af8a05ad4d570e62d6d1e394037d --- /dev/null +++ b/arch/loongarch/boot/tools/relocs.c @@ -0,0 +1,568 @@ +// SPDX-License-Identifier: GPL-2.0 +/* This is included from relocs_32/64.c */ + +#define ElfW(type) _ElfW(ELF_BITS, type) +#define _ElfW(bits, type) __ElfW(bits, type) +#define __ElfW(bits, type) Elf##bits##_##type + +#define Elf_Addr ElfW(Addr) +#define Elf_Ehdr ElfW(Ehdr) +#define Elf_Phdr ElfW(Phdr) +#define Elf_Shdr ElfW(Shdr) +#define Elf_Sym ElfW(Sym) + +static Elf_Ehdr ehdr; + +struct relocs { + uint32_t *offset; + unsigned long count; + unsigned long size; +}; + +static struct relocs relocs; + +struct section { + Elf_Shdr shdr; + struct section *link; + Elf_Sym *symtab; + Elf_Rel *reltab; + char *strtab; + long shdr_offset; +}; +static struct section *secs; + +static const char * const regex_sym_kernel = { +/* Symbols matching these regex's should never be relocated */ + "^(__crc_)", +}; + +static regex_t sym_regex_c; + +static int regex_skip_reloc(const char *sym_name) +{ + return !regexec(&sym_regex_c, sym_name, 0, NULL, 0); +} + +static void regex_init(void) +{ + char errbuf[128]; + int err; + + err = regcomp(&sym_regex_c, regex_sym_kernel, + REG_EXTENDED|REG_NOSUB); + + if (err) { + regerror(err, &sym_regex_c, errbuf, sizeof(errbuf)); + die("%s", errbuf); + } +} + +static const char *rel_type(unsigned type) +{ + static const char * const type_name[] = { +#define REL_TYPE(X)[X] = #X + REL_TYPE(R_LARCH_NONE), + REL_TYPE(R_LARCH_32), + REL_TYPE(R_LARCH_64), + REL_TYPE(R_LARCH_MARK_LA), + REL_TYPE(R_LARCH_MARK_PCREL), + REL_TYPE(R_LARCH_SOP_PUSH_PCREL), + REL_TYPE(R_LARCH_SOP_PUSH_ABSOLUTE), + REL_TYPE(R_LARCH_SOP_PUSH_DUP), + REL_TYPE(R_LARCH_SOP_PUSH_PLT_PCREL), + REL_TYPE(R_LARCH_SOP_SUB), + REL_TYPE(R_LARCH_SOP_SL), + REL_TYPE(R_LARCH_SOP_SR), + REL_TYPE(R_LARCH_SOP_ADD), + REL_TYPE(R_LARCH_SOP_AND), + REL_TYPE(R_LARCH_SOP_IF_ELSE), + REL_TYPE(R_LARCH_SOP_POP_32_S_10_5), + REL_TYPE(R_LARCH_SOP_POP_32_U_10_12), + REL_TYPE(R_LARCH_SOP_POP_32_S_10_12), + REL_TYPE(R_LARCH_SOP_POP_32_S_10_16), + REL_TYPE(R_LARCH_SOP_POP_32_S_10_16_S2), + REL_TYPE(R_LARCH_SOP_POP_32_S_5_20), + REL_TYPE(R_LARCH_SOP_POP_32_S_0_5_10_16_S2), + REL_TYPE(R_LARCH_SOP_POP_32_S_0_10_10_16_S2), + REL_TYPE(R_LARCH_SOP_POP_32_U), + REL_TYPE(R_LARCH_ADD32), + REL_TYPE(R_LARCH_ADD64), + REL_TYPE(R_LARCH_SUB32), + REL_TYPE(R_LARCH_SUB64), +#undef REL_TYPE + }; + const char *name = "unknown type rel type name"; + + if (type < ARRAY_SIZE(type_name) && type_name[type]) + name = type_name[type]; + return name; +} + +static const char *sec_name(unsigned shndx) +{ + const char *sec_strtab; + const char *name; + + sec_strtab = secs[ehdr.e_shstrndx].strtab; + if (shndx < ehdr.e_shnum) + name = sec_strtab + secs[shndx].shdr.sh_name; + else if (shndx == SHN_ABS) + name = "ABSOLUTE"; + else if (shndx == SHN_COMMON) + name = "COMMON"; + else + name = ""; + return name; +} + +static struct section *sec_lookup(const char *secname) +{ + int i; + + for (i = 0; i < ehdr.e_shnum; i++) + if (strcmp(secname, sec_name(i)) == 0) + return &secs[i]; + + return NULL; +} + +static const char *sym_name(const char *sym_strtab, Elf_Sym *sym) +{ + const char *name; + + if (sym->st_name) + name = sym_strtab + sym->st_name; + else + name = sec_name(sym->st_shndx); + return name; +} + +static void read_ehdr(FILE *fp) +{ + if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1) + die("Cannot read ELF header: %s\n", strerror(errno)); + + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + die("No ELF magic\n"); + + if (ehdr.e_ident[EI_CLASS] != ELF_CLASS) + die("Not a %d bit executable\n", ELF_BITS); + + if (ehdr.e_ident[EI_DATA] != ELFDATA2LSB) + die("Unsupported ELF Endianness\n"); + + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + die("Unknown ELF version\n"); + + if ((ehdr.e_type != ET_EXEC) && (ehdr.e_type != ET_DYN)) + die("Unsupported ELF header type\n"); + + if (ehdr.e_machine != ELF_MACHINE) + die("Not for %s\n", ELF_MACHINE_NAME); + + if (ehdr.e_version != EV_CURRENT) + die("Unknown ELF version\n"); + + if (ehdr.e_ehsize != sizeof(Elf_Ehdr)) + die("Bad Elf header size\n"); + + if (ehdr.e_phentsize != sizeof(Elf_Phdr)) + die("Bad program header entry\n"); + + if (ehdr.e_shentsize != sizeof(Elf_Shdr)) + die("Bad section header entry\n"); + + if (ehdr.e_shstrndx >= ehdr.e_shnum) + die("String table index out of bounds\n"); +} + +static void read_shdrs(FILE *fp) +{ + int i; + + secs = calloc(ehdr.e_shnum, sizeof(struct section)); + if (!secs) + die("Unable to allocate %d section headers\n", ehdr.e_shnum); + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", ehdr.e_shoff, strerror(errno)); + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + sec->shdr_offset = ftell(fp); + if (fread(&sec->shdr, sizeof(Elf_Shdr), 1, fp) != 1) + die("Cannot read ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + + if (sec->shdr.sh_link < ehdr.e_shnum) + sec->link = &secs[sec->shdr.sh_link]; + } +} + +static void read_strtabs(FILE *fp) +{ + int i; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_STRTAB) + continue; + + sec->strtab = malloc(sec->shdr.sh_size); + if (!sec->strtab) + die("malloc of %d bytes for strtab failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->strtab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + } +} + +static void read_symtabs(FILE *fp) +{ + int i; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + if (sec->shdr.sh_type != SHT_SYMTAB) + continue; + + sec->symtab = malloc(sec->shdr.sh_size); + if (!sec->symtab) + die("malloc of %d bytes for symtab failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + } +} + +static void read_relocs(FILE *fp) +{ + static unsigned long base = 0; + int i, j; + + if (!base) { + struct section *sec = sec_lookup(".text"); + + if (!sec) + die("Could not find .text section\n"); + + base = sec->shdr.sh_addr; + } + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec->reltab = malloc(sec->shdr.sh_size); + if (!sec->reltab) + die("malloc of %d bytes for relocs failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->reltab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + /* Set offset into kernel image */ + rel->r_offset -= base; + } + } +} + +static void remove_relocs(FILE *fp) +{ + int i; + Elf_Shdr shdr; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + if (fseek(fp, sec->shdr_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr_offset, strerror(errno)); + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot read ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + + /* Set relocation section size to 0, effectively removing it. + * This is necessary due to lack of support for relocations + * in objcopy when creating 32bit elf from 64bit elf. + */ + shdr.sh_size = 0; + + if (fseek(fp, sec->shdr_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr_offset, strerror(errno)); + + if (fwrite(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot write ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + } +} + +static void add_reloc(struct relocs *r, uint32_t offset, unsigned type) +{ + /* Relocation representation in binary table: + * |76543210|76543210|76543210|76543210| + * | Type | offset from _text >> 2 | + */ + offset >>= 2; + if (offset > 0x00FFFFFF) + die("Kernel image exceeds maximum size for relocation!\n"); + + offset = (offset & 0x00FFFFFF) | ((type & 0xFF) << 24); + + if (r->count == r->size) { + unsigned long newsize = r->size + 50000; + void *mem = realloc(r->offset, newsize * sizeof(r->offset[0])); + + if (!mem) + die("realloc failed\n"); + + r->offset = mem; + r->size = newsize; + } + r->offset[r->count++] = offset; +} + +static void walk_relocs(int (*process)(struct section *sec, Elf_Rel *rel, + Elf_Sym *sym, const char *symname)) +{ + int i; + + /* Walk through the relocations */ + for (i = 0; i < ehdr.e_shnum; i++) { + char *sym_strtab; + Elf_Sym *sh_symtab; + struct section *sec_applies, *sec_symtab; + int j; + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec_symtab = sec->link; + sec_applies = &secs[sec->shdr.sh_info]; + if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) + continue; + + sh_symtab = sec_symtab->symtab; + sym_strtab = sec_symtab->link->strtab; + for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + Elf_Sym *sym = &sh_symtab[ELF_R_SYM(rel->r_info)]; + const char *symname = sym_name(sym_strtab, sym); + + process(sec, rel, sym, symname); + } + } +} + +static int do_reloc(struct section *sec, Elf_Rel *rel, Elf_Sym *sym, + const char *symname) +{ + unsigned r_type = ELF_R_TYPE(rel->r_info); + unsigned bind = ELF_ST_BIND(sym->st_info); + + if ((bind == STB_WEAK) && (sym->st_value == 0)) { + /* Don't relocate weak symbols without a target */ + return 0; + } + + if (regex_skip_reloc(symname)) + return 0; + + switch (r_type) { + case R_LARCH_NONE: + case R_LARCH_MARK_PCREL: + case R_LARCH_SOP_PUSH_PCREL: + case R_LARCH_SOP_PUSH_ABSOLUTE: + case R_LARCH_SOP_PUSH_DUP: + case R_LARCH_SOP_PUSH_PLT_PCREL: + case R_LARCH_SOP_SUB: + case R_LARCH_SOP_SL: + case R_LARCH_SOP_SR: + case R_LARCH_SOP_ADD: + case R_LARCH_SOP_AND: + case R_LARCH_SOP_IF_ELSE: + case R_LARCH_SOP_POP_32_S_10_5: + case R_LARCH_SOP_POP_32_U_10_12: + case R_LARCH_SOP_POP_32_S_10_12: + case R_LARCH_SOP_POP_32_S_10_16: + case R_LARCH_SOP_POP_32_S_10_16_S2: + case R_LARCH_SOP_POP_32_S_5_20: + case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: + case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: + case R_LARCH_SOP_POP_32_U: + case R_LARCH_ADD32: + case R_LARCH_ADD64: + case R_LARCH_SUB32: + case R_LARCH_SUB64: + break; + + case R_LARCH_32: + case R_LARCH_64: + case R_LARCH_MARK_LA: + add_reloc(&relocs, rel->r_offset, r_type); + break; + + default: + die("Unsupported relocation type: %s (%d)\n", rel_type(r_type), r_type); + break; + } + + return 0; +} + +static int write_reloc_as_bin(uint32_t v, FILE *f) +{ + return fwrite(&v, 4, 1, f); +} + +static int write_reloc_as_text(uint32_t v, FILE *f) +{ + int res; + + res = fprintf(f, "\t.long 0x%08"PRIx32"\n", v); + if (res < 0) + return res; + else + return sizeof(uint32_t); +} + +static void emit_relocs(int as_text, int as_bin, FILE *outf) +{ + int i; + int (*write_reloc)(uint32_t, FILE *) = write_reloc_as_bin; + int size = 0; + int size_reserved; + struct section *sec_reloc; + + sec_reloc = sec_lookup(".data.reloc"); + if (!sec_reloc) + die("Could not find relocation section\n"); + + size_reserved = sec_reloc->shdr.sh_size; + + /* Collect up the relocations */ + walk_relocs(do_reloc); + + /* Print the relocations */ + if (as_text) { + /* Print the relocations in a form suitable that + * gas will like. + */ + printf(".section \".data.reloc\",\"a\"\n"); + printf(".balign 4\n"); + /* Output text to stdout */ + write_reloc = write_reloc_as_text; + outf = stdout; + } else if (as_bin) { + /* Output raw binary to stdout */ + outf = stdout; + } else { + /* Seek to offset of the relocation section. + * Each relocation is then written into the + * vmlinux kernel image. + */ + if (fseek(outf, sec_reloc->shdr.sh_offset, SEEK_SET) < 0) { + die("Seek to %d failed: %s\n", + sec_reloc->shdr.sh_offset, strerror(errno)); + } + } + + for (i = 0; i < relocs.count; i++) + size += write_reloc(relocs.offset[i], outf); + + /* Print a stop, but only if we've actually written some relocs */ + if (size) + size += write_reloc(0, outf); + + if (size > size_reserved) + /* Die, but suggest a value for CONFIG_RELOCATION_TABLE_SIZE + * which will fix this problem and allow a bit of headroom + * if more kernel features are enabled + */ + die("Relocations overflow available space!\n" \ + "Please adjust CONFIG_RELOCATION_TABLE_SIZE " \ + "to at least 0x%08x\n", (size + 0x1000) & ~0xFFF); +} + +/* + * As an aid to debugging problems with different linkers + * print summary information about the relocs. + * Since different linkers tend to emit the sections in + * different orders we use the section names in the output. + */ +static int do_reloc_info(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym, + const char *symname) +{ + printf("%-16s 0x%08x %-35s %-40s %-16s\n", + sec_name(sec->shdr.sh_info), + (unsigned int)rel->r_offset, + rel_type(ELF_R_TYPE(rel->r_info)), + symname, + sec_name(sym->st_shndx)); + return 0; +} + +static void print_reloc_info(void) +{ + printf("%-16s %-10s %-35s %-40s %-16s\n", + "reloc section", + "offset", + "reloc type", + "symbol", + "symbol section"); + walk_relocs(do_reloc_info); +} + +#if ELF_BITS == 64 +# define process process_64 +#else +# define process process_32 +#endif + +void process(FILE *fp, int as_text, int as_bin, + int show_reloc_info, int keep_relocs) +{ + regex_init(); + read_ehdr(fp); + read_shdrs(fp); + read_strtabs(fp); + read_symtabs(fp); + read_relocs(fp); + if (show_reloc_info) { + print_reloc_info(); + return; + } + emit_relocs(as_text, as_bin, fp); + if (!keep_relocs) + remove_relocs(fp); +} diff --git a/arch/loongarch/boot/tools/relocs.h b/arch/loongarch/boot/tools/relocs.h new file mode 100644 index 0000000000000000000000000000000000000000..483091f112eed4c990b26a2aa7fb311323919f13 --- /dev/null +++ b/arch/loongarch/boot/tools/relocs.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef RELOCS_H +#define RELOCS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void die(char *fmt, ...); + +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 +#endif +#define R_LARCH_NONE 0 +#define R_LARCH_32 1 +#define R_LARCH_64 2 +#define R_LARCH_MARK_LA 20 +#define R_LARCH_MARK_PCREL 21 +#define R_LARCH_SOP_PUSH_PCREL 22 +#define R_LARCH_SOP_PUSH_ABSOLUTE 23 +#define R_LARCH_SOP_PUSH_DUP 24 +#define R_LARCH_SOP_PUSH_PLT_PCREL 29 +#define R_LARCH_SOP_SUB 32 +#define R_LARCH_SOP_SL 33 +#define R_LARCH_SOP_SR 34 +#define R_LARCH_SOP_ADD 35 +#define R_LARCH_SOP_AND 36 +#define R_LARCH_SOP_IF_ELSE 37 +#define R_LARCH_SOP_POP_32_S_10_5 38 +#define R_LARCH_SOP_POP_32_U_10_12 39 +#define R_LARCH_SOP_POP_32_S_10_12 40 +#define R_LARCH_SOP_POP_32_S_10_16 41 +#define R_LARCH_SOP_POP_32_S_10_16_S2 42 +#define R_LARCH_SOP_POP_32_S_5_20 43 +#define R_LARCH_SOP_POP_32_S_0_5_10_16_S2 44 +#define R_LARCH_SOP_POP_32_S_0_10_10_16_S2 45 +#define R_LARCH_SOP_POP_32_U 46 +#define R_LARCH_ADD32 50 +#define R_LARCH_ADD64 51 +#define R_LARCH_SUB32 55 +#define R_LARCH_SUB64 56 + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +void process_32(FILE *fp, int as_text, int as_bin, + int show_reloc_info, int keep_relocs); +void process_64(FILE *fp, int as_text, int as_bin, + int show_reloc_info, int keep_relocs); + +#endif /* RELOCS_H */ diff --git a/arch/loongarch/boot/tools/relocs_32.c b/arch/loongarch/boot/tools/relocs_32.c new file mode 100644 index 0000000000000000000000000000000000000000..7cae3d8403bb73dd7def61c6ca64a4a452471986 --- /dev/null +++ b/arch/loongarch/boot/tools/relocs_32.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "relocs.h" + +#define ELF_BITS 32 + +#define ELF_MACHINE EM_LOONGARCH +#define ELF_MACHINE_NAME "LOONGARCH32" +#define SHT_REL_TYPE SHT_RELA +#define Elf_Rel ElfW(Rela) + +#define ELF_CLASS ELFCLASS32 +#define ELF_R_SYM(val) ELF32_R_SYM(val) +#define ELF_R_TYPE(val) ELF32_R_TYPE(val) +#define ELF_ST_TYPE(o) ELF32_ST_TYPE(o) +#define ELF_ST_BIND(o) ELF32_ST_BIND(o) +#define ELF_ST_VISIBILITY(o) ELF32_ST_VISIBILITY(o) + +#include "relocs.c" diff --git a/arch/loongarch/boot/tools/relocs_64.c b/arch/loongarch/boot/tools/relocs_64.c new file mode 100644 index 0000000000000000000000000000000000000000..1024133af8ec7aa4e5525c9e1817b85bc973f295 --- /dev/null +++ b/arch/loongarch/boot/tools/relocs_64.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "relocs.h" + +#define ELF_BITS 64 + +#define ELF_MACHINE EM_LOONGARCH +#define ELF_MACHINE_NAME "LOONGARCH64" +#define SHT_REL_TYPE SHT_RELA +#define Elf_Rel Elf64_Rela + +#define ELF_CLASS ELFCLASS64 +#define ELF_R_SYM(val) ELF64_R_SYM(val) +#define ELF_R_TYPE(val) ELF64_R_TYPE(val) +#define ELF_ST_TYPE(o) ELF64_ST_TYPE(o) +#define ELF_ST_BIND(o) ELF64_ST_BIND(o) +#define ELF_ST_VISIBILITY(o) ELF64_ST_VISIBILITY(o) + +#include "relocs.c" diff --git a/arch/loongarch/boot/tools/relocs_main.c b/arch/loongarch/boot/tools/relocs_main.c new file mode 100644 index 0000000000000000000000000000000000000000..e2453a564b119e05c4b276c7a98a7dae0ab4b90a --- /dev/null +++ b/arch/loongarch/boot/tools/relocs_main.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "relocs.h" + +void die(char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + exit(1); +} + +static void usage(void) +{ + die("relocs [--reloc-info|--text|--bin|--keep] vmlinux\n"); +} + +int main(int argc, char **argv) +{ + int show_reloc_info, as_text, as_bin, keep_relocs; + const char *fname; + FILE *fp; + int i; + unsigned char e_ident[EI_NIDENT]; + + show_reloc_info = 0; + as_text = 0; + as_bin = 0; + keep_relocs = 0; + fname = NULL; + for (i = 1; i < argc; i++) { + char *arg = argv[i]; + + if (*arg == '-') { + if (strcmp(arg, "--reloc-info") == 0) { + show_reloc_info = 1; + continue; + } + if (strcmp(arg, "--text") == 0) { + as_text = 1; + continue; + } + if (strcmp(arg, "--bin") == 0) { + as_bin = 1; + continue; + } + if (strcmp(arg, "--keep") == 0) { + keep_relocs = 1; + continue; + } + } else if (!fname) { + fname = arg; + continue; + } + usage(); + } + if (!fname) + usage(); + + fp = fopen(fname, "r+"); + if (!fp) + die("Cannot open %s: %s\n", fname, strerror(errno)); + + if (fread(&e_ident, 1, EI_NIDENT, fp) != EI_NIDENT) + die("Cannot read %s: %s", fname, strerror(errno)); + + rewind(fp); + if (e_ident[EI_CLASS] == ELFCLASS64) + process_64(fp, as_text, as_bin, show_reloc_info, keep_relocs); + else + process_32(fp, as_text, as_bin, show_reloc_info, keep_relocs); + fclose(fp); + return 0; +} diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig new file mode 100644 index 0000000000000000000000000000000000000000..9e894edabd00f5bd1fb61f45a7f2ea683625d461 --- /dev/null +++ b/arch/loongarch/configs/loongson3_defconfig @@ -0,0 +1,950 @@ +# CONFIG_WERROR is not set +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZMA=y +CONFIG_SYSVIPC=y +CONFIG_LOONGARCH=y +CONFIG_MACH_LOONGSON64=y +CONFIG_CPU_LOONGSON64=y +CONFIG_64BIT=y +CONFIG_DMI=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_PREEMPT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_LOG_BUF_SHIFT=16 +CONFIG_NUMA_BALANCING=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RT_SOFTINT_OPTIMIZATION=y +CONFIG_SYSFS_DEPRECATED=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_BPF_SYSCALL=y +CONFIG_EXPERT=y +CONFIG_USERFAULTFD=y +CONFIG_PERF_EVENTS=y +# CONFIG_COMPAT_BRK is not set +CONFIG_PAGE_SIZE_16KB=y +CONFIG_CPU_HAS_LSX=y +CONFIG_CPU_HAS_LASX=y +CONFIG_NUMA=y +CONFIG_EFI=y +CONFIG_SMP=y +CONFIG_NR_CPUS=256 +CONFIG_HZ_250=y +CONFIG_KEXEC=y +CONFIG_HIBERNATION=y +CONFIG_ACPI=y +CONFIG_ACPI_SPCR_TABLE=y +CONFIG_ACPI_HOTPLUG_CPU=y +CONFIG_ACPI_TAD=y +CONFIG_ACPI_DOCK=y +CONFIG_ACPI_IPMI=m +CONFIG_ACPI_PCI_SLOT=y +CONFIG_ACPI_HOTPLUG_MEMORY=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER=y +CONFIG_EFI_BOOTLOADER_CONTROL=m +CONFIG_EFI_CAPSULE_LOADER=m +CONFIG_EFI_TEST=m +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_PARAVIRT_SPINLOCKS=y +CONFIG_PARAVIRT_TIME_ACCOUNTING=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_BLK_DEV_ZONED=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_DEV_THROTTLING_LOW=y +CONFIG_BLK_WBT=y +CONFIG_BLK_CGROUP_IOLATENCY=y +CONFIG_BLK_CGROUP_IOCOST=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_BSD_DISKLABEL=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_CMDLINE_PARTITION=y +CONFIG_IOSCHED_BFQ=y +CONFIG_BFQ_GROUP_IOSCHED=y +CONFIG_GKI_HACKS_TO_FIX=y +CONFIG_BINFMT_MISC=m +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_KSM=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_FRONTSWAP=y +CONFIG_CMA=y +CONFIG_ZSWAP=y +CONFIG_ZSWAP_COMPRESSOR_DEFAULT_ZSTD=y +CONFIG_ZPOOL=y +CONFIG_ZBUD=y +CONFIG_Z3FOLD=y +CONFIG_ZSMALLOC=m +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_MROUTE_MULTIPLE_TABLES=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_INET_ESP=m +CONFIG_INET_UDP_DIAG=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_BBR=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_INET6_ESP=m +CONFIG_IPV6_MROUTE=y +CONFIG_MPTCP=y +CONFIG_NETWORK_PHY_TIMESTAMPING=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NETFILTER_NETLINK_LOG=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_TABLES=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_NFCT=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BPFILTER=y +CONFIG_IP_SCTP=m +CONFIG_RDS=y +CONFIG_L2TP=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_BPF=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_NETLINK_DIAG=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +CONFIG_BPF_STREAM_PARSER=y +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_HS=y +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIBTUSB_MTK=y +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_BCSP=y +CONFIG_BT_HCIUART_ATH3K=y +CONFIG_BT_HCIUART_INTEL=y +CONFIG_BT_HCIUART_AG6XX=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIDTL1=m +CONFIG_BT_HCIBT3C=m +CONFIG_BT_HCIBLUECARD=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_ATH3K=m +CONFIG_CFG80211=m +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_RFKILL=m +CONFIG_RFKILL_INPUT=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_CEPH_LIB=m +CONFIG_PCIEPORTBUS=y +CONFIG_HOTPLUG_PCI_PCIE=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_IOV=y +CONFIG_HOTPLUG_PCI=y +CONFIG_HOTPLUG_PCI_SHPC=y +CONFIG_PCI_HOST_GENERIC=y +CONFIG_PCCARD=m +CONFIG_YENTA=m +CONFIG_RAPIDIO=y +CONFIG_RAPIDIO_TSI721=y +CONFIG_RAPIDIO_ENABLE_RX_TX_PORTS=y +CONFIG_RAPIDIO_ENUM_BASIC=m +CONFIG_RAPIDIO_CHMAN=m +CONFIG_RAPIDIO_MPORT_CDEV=m +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_MTD=m +CONFIG_MTD_BLOCK=m +CONFIG_MTD_CFI=m +CONFIG_MTD_JEDECPROBE=m +CONFIG_MTD_CFI_INTELEXT=m +CONFIG_MTD_CFI_AMDSTD=m +CONFIG_MTD_CFI_STAA=m +CONFIG_MTD_RAM=m +CONFIG_MTD_ROM=m +CONFIG_MTD_UBI=m +CONFIG_MTD_UBI_BLOCK=y +CONFIG_PARPORT=y +CONFIG_PARPORT_PC=y +CONFIG_PARPORT_SERIAL=y +CONFIG_PARPORT_PC_FIFO=y +CONFIG_ZRAM=m +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_CRYPTOLOOP=y +CONFIG_BLK_DEV_DRBD=m +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_RBD=m +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_EEPROM_AT24=m +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=m +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SPI_ATTRS=m +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_ISCSI_TCP=m +CONFIG_SCSI_MVSAS=y +# CONFIG_SCSI_MVSAS_DEBUG is not set +CONFIG_SCSI_MVSAS_TASKLET=y +CONFIG_SCSI_MVUMI=y +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=y +CONFIG_MEGARAID_MAILBOX=y +CONFIG_MEGARAID_LEGACY=y +CONFIG_MEGARAID_SAS=y +CONFIG_SCSI_MPT2SAS=y +CONFIG_LIBFC=m +CONFIG_LIBFCOE=m +CONFIG_FCOE=m +CONFIG_SCSI_QLOGIC_1280=m +CONFIG_SCSI_QLA_FC=m +CONFIG_TCM_QLA2XXX=m +CONFIG_SCSI_QLA_ISCSI=m +CONFIG_SCSI_LPFC=m +CONFIG_SCSI_VIRTIO=m +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_PATA_ATIIXP=y +CONFIG_PATA_PCMCIA=m +CONFIG_MD=y +CONFIG_BLK_DEV_MD=m +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_MD_MULTIPATH=m +CONFIG_BCACHE=m +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_MIRROR=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_NETDEVICES=y +CONFIG_BONDING=m +CONFIG_DUMMY=y +CONFIG_WIREGUARD=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_IPVLAN=m +CONFIG_VXLAN=y +CONFIG_RIONET=m +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_VIRTIO_NET=m +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_ADAPTEC is not set +# CONFIG_NET_VENDOR_AGERE is not set +# CONFIG_NET_VENDOR_ALACRITECH is not set +# CONFIG_NET_VENDOR_ALTEON is not set +# CONFIG_NET_VENDOR_AMAZON is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_AQUANTIA is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_ATHEROS is not set +CONFIG_BNX2=y +# CONFIG_NET_VENDOR_BROCADE is not set +# CONFIG_NET_VENDOR_CAVIUM is not set +CONFIG_CHELSIO_T1=m +CONFIG_CHELSIO_T1_1G=y +CONFIG_CHELSIO_T3=m +CONFIG_CHELSIO_T4=m +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_CISCO is not set +# CONFIG_NET_VENDOR_DEC is not set +# CONFIG_NET_VENDOR_DLINK is not set +# CONFIG_NET_VENDOR_EMULEX is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +# CONFIG_NET_VENDOR_I825XX is not set +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IXGB=y +CONFIG_IXGBE=y +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MELLANOX is not set +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MYRI is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_NETRONOME is not set +# CONFIG_NET_VENDOR_NVIDIA is not set +# CONFIG_NET_VENDOR_OKI is not set +# CONFIG_NET_VENDOR_QLOGIC is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RDC is not set +CONFIG_8139CP=m +CONFIG_8139TOO=m +CONFIG_R8169=y +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set +# CONFIG_NET_VENDOR_SILAN is not set +# CONFIG_NET_VENDOR_SIS is not set +# CONFIG_NET_VENDOR_SMSC is not set +CONFIG_STMMAC_ETH=y +# CONFIG_NET_VENDOR_SUN is not set +# CONFIG_NET_VENDOR_TEHUTI is not set +# CONFIG_NET_VENDOR_TI is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_NET_VENDOR_XILINX is not set +CONFIG_PPP=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +# CONFIG_USB_NET_AX8817X is not set +# CONFIG_USB_NET_AX88179_178A is not set +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +# CONFIG_USB_NET_NET1080 is not set +# CONFIG_USB_BELKIN is not set +# CONFIG_USB_ARMLINUX is not set +# CONFIG_USB_NET_ZAURUS is not set +CONFIG_ATH9K=m +CONFIG_ATH9K_HTC=m +CONFIG_IWLWIFI=m +CONFIG_IWLDVM=m +CONFIG_IWLMVM=m +CONFIG_IWLWIFI_BCAST_FILTERING=y +CONFIG_HOSTAP=m +CONFIG_MT7601U=m +CONFIG_RT2X00=m +CONFIG_RT2800USB=m +CONFIG_RTL8192CE=m +CONFIG_RTL8192SE=m +CONFIG_RTL8192DE=m +CONFIG_RTL8723AE=m +CONFIG_RTL8723BE=m +CONFIG_RTL8188EE=m +CONFIG_RTL8192EE=m +CONFIG_RTL8821AE=m +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8822BE=m +CONFIG_RTW88_8822CE=m +CONFIG_RTW88_8723DE=m +CONFIG_RTW88_8821CE=m +CONFIG_ZD1211RW=m +CONFIG_USB_NET_RNDIS_WLAN=m +CONFIG_USB4_NET=m +CONFIG_INPUT_POLLDEV=m +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_MOUSEDEV_PSAUX=y +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_XTKBD=m +CONFIG_MOUSE_PS2_ELANTECH=y +CONFIG_MOUSE_PS2_SENTELIC=y +CONFIG_MOUSE_SERIAL=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=m +CONFIG_SERIO_SERPORT=m +CONFIG_SERIO_RAW=m +CONFIG_LEGACY_PTY_COUNT=16 +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=16 +CONFIG_SERIAL_8250_RUNTIME_UARTS=16 +CONFIG_SERIAL_8250_EXTENDED=y +CONFIG_SERIAL_8250_MANY_PORTS=y +CONFIG_SERIAL_8250_SHARE_IRQ=y +CONFIG_SERIAL_8250_RSA=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_NONSTANDARD=y +CONFIG_PRINTER=m +CONFIG_VIRTIO_CONSOLE=y +CONFIG_IPMI_HANDLER=m +CONFIG_IPMI_DEVICE_INTERFACE=m +CONFIG_IPMI_SI=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=m +CONFIG_RAW_DRIVER=m +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_PIIX4=y +CONFIG_I2C_GPIO=y +CONFIG_SPI=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_LOONGSON=y +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM93=m +CONFIG_SENSORS_W83795=m +CONFIG_SENSORS_W83627HF=m +CONFIG_RC_CORE=m +CONFIG_LIRC=y +CONFIG_RC_DECODERS=y +CONFIG_IR_NEC_DECODER=m +CONFIG_IR_RC5_DECODER=m +CONFIG_IR_RC6_DECODER=m +CONFIG_IR_JVC_DECODER=m +CONFIG_IR_SONY_DECODER=m +CONFIG_IR_SANYO_DECODER=m +CONFIG_IR_SHARP_DECODER=m +CONFIG_IR_MCE_KBD_DECODER=m +CONFIG_IR_XMP_DECODER=m +CONFIG_IR_IMON_DECODER=m +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_VIDEO_CLASS=m +CONFIG_MEDIA_PCI_SUPPORT=y +CONFIG_VIDEO_BT848=m +CONFIG_DVB_BT8XX=m +# CONFIG_DVB_LGS8GL5 is not set +CONFIG_DRM=y +CONFIG_DRM_RADEON=m +CONFIG_DRM_RADEON_USERPTR=y +CONFIG_DRM_AMDGPU=m +CONFIG_DRM_AMDGPU_SI=y +CONFIG_DRM_AMDGPU_CIK=y +CONFIG_DRM_AMDGPU_USERPTR=y +CONFIG_DRM_AST=y +CONFIG_DRM_LOONGSON=y +CONFIG_DRM_QXL=m +CONFIG_DRM_VIRTIO_GPU=m +CONFIG_FB_EFI=y +CONFIG_FB_RADEON=y +CONFIG_LCD_PLATFORM=m +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +# CONFIG_SND_ISA is not set +CONFIG_SND_BT87X=m +CONFIG_SND_BT87X_OVERCLOCK=y +CONFIG_SND_HDA_INTEL=y +CONFIG_SND_HDA_LOONGSON=y +CONFIG_SND_HDA_HWDEP=y +CONFIG_SND_HDA_INPUT_BEEP=y +CONFIG_SND_HDA_PATCH_LOADER=y +CONFIG_SND_HDA_CODEC_REALTEK=y +CONFIG_SND_HDA_CODEC_SIGMATEL=y +CONFIG_SND_HDA_CODEC_HDMI=y +CONFIG_SND_HDA_CODEC_CONEXANT=y +CONFIG_SND_USB_AUDIO=m +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_CHERRY=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_SUNPLUS=m +CONFIG_USB_HIDDEV=y +CONFIG_USB=y +CONFIG_USB_OTG=y +CONFIG_USB_MON=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_OHCI_HCD_PLATFORM=y +CONFIG_USB_UHCI_HCD=m +CONFIG_USB_ACM=m +CONFIG_USB_PRINTER=m +CONFIG_USB_STORAGE=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_UAS=m +CONFIG_USB_DWC2=y +CONFIG_USB_DWC2_HOST=y +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_GADGET=y +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_TCPCI=m +CONFIG_TYPEC_UCSI=m +CONFIG_UCSI_ACPI=m +CONFIG_INFINIBAND=m +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_EFI=y +CONFIG_DMADEVICES=y +CONFIG_UDMABUF=y +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_DEFERRED_FREE=y +CONFIG_DMABUF_HEAPS_PAGE_POOL=y +CONFIG_DMABUF_HEAPS_SYSTEM=m +CONFIG_DMABUF_HEAPS_CMA=m +CONFIG_UIO=m +CONFIG_UIO_PDRV_GENIRQ=m +CONFIG_UIO_DMEM_GENIRQ=m +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VFIO=m +CONFIG_VFIO_PCI=m +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=m +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=m +CONFIG_STAGING=y +CONFIG_COMEDI=m +CONFIG_COMEDI_PCI_DRIVERS=m +CONFIG_COMEDI_8255_PCI=m +CONFIG_COMEDI_ADL_PCI6208=m +CONFIG_COMEDI_ADL_PCI7X3X=m +CONFIG_COMEDI_ADL_PCI8164=m +CONFIG_COMEDI_ADL_PCI9111=m +CONFIG_COMEDI_ADL_PCI9118=m +CONFIG_COMEDI_ADV_PCI1710=m +CONFIG_COMEDI_ADV_PCI1720=m +CONFIG_COMEDI_ADV_PCI1723=m +CONFIG_COMEDI_ADV_PCI1724=m +CONFIG_COMEDI_ADV_PCI1760=m +CONFIG_COMEDI_ADV_PCI_DIO=m +CONFIG_COMEDI_NI_LABPC_PCI=m +CONFIG_COMEDI_NI_PCIDIO=m +CONFIG_COMEDI_NI_PCIMIO=m +CONFIG_R8188EU=m +# CONFIG_88EU_AP_MODE is not set +CONFIG_ASHMEM=y +CONFIG_ION=y +CONFIG_ION_SYSTEM_HEAP=m +CONFIG_ION_CMA_HEAP=m +CONFIG_LOONGSON_IOMMU=y +CONFIG_PM_DEVFREQ=y +CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y +CONFIG_DEVFREQ_GOV_PERFORMANCE=y +CONFIG_DEVFREQ_GOV_POWERSAVE=y +CONFIG_DEVFREQ_GOV_USERSPACE=y +CONFIG_NTB=m +CONFIG_NTB_MSI=y +CONFIG_NTB_IDT=m +CONFIG_NTB_SWITCHTEC=m +CONFIG_NTB_PERF=m +CONFIG_NTB_TRANSPORT=m +CONFIG_PWM=y +CONFIG_USB4=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ANDROID_BINDERFS=y +CONFIG_EXT2_FS=y +CONFIG_EXT2_FS_XATTR=y +CONFIG_EXT2_FS_POSIX_ACL=y +CONFIG_EXT2_FS_SECURITY=y +CONFIG_EXT3_FS=y +CONFIG_EXT3_FS_POSIX_ACL=y +CONFIG_EXT3_FS_SECURITY=y +CONFIG_JFS_FS=m +CONFIG_JFS_POSIX_ACL=y +CONFIG_JFS_SECURITY=y +CONFIG_XFS_FS=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_GFS2_FS=m +CONFIG_GFS2_FS_LOCKING_DLM=y +CONFIG_OCFS2_FS=m +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +# CONFIG_PRINT_QUOTA_WARNING is not set +CONFIG_QFMT_V1=m +CONFIG_QFMT_V2=m +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=m +CONFIG_CUSE=m +CONFIG_VIRTIO_FS=m +CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_INDEX=y +CONFIG_OVERLAY_FS_XINO_AUTO=y +CONFIG_OVERLAY_FS_METACOPY=y +CONFIG_FSCACHE=y +CONFIG_CACHEFILES=m +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=m +CONFIG_VFAT_FS=m +CONFIG_FAT_DEFAULT_CODEPAGE=936 +CONFIG_FAT_DEFAULT_IOCHARSET="gb2312" +CONFIG_EXFAT_FS=m +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +CONFIG_ORANGEFS_FS=m +CONFIG_ECRYPT_FS=m +CONFIG_ECRYPT_FS_MESSAGING=y +CONFIG_HFS_FS=m +CONFIG_HFSPLUS_FS=m +CONFIG_UBIFS_FS=m +CONFIG_UBIFS_FS_ADVANCED_COMPR=y +CONFIG_CRAMFS=m +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_MINIX_FS=m +CONFIG_ROMFS_FS=m +CONFIG_SYSV_FS=m +CONFIG_UFS_FS=m +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_ZIP_LZMA=y +CONFIG_EROFS_FS_PCPU_KTHREAD=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y +CONFIG_NFSD=y +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_CEPH_FS=m +CONFIG_CEPH_FSCACHE=y +CONFIG_CEPH_FS_POSIX_ACL=y +CONFIG_CEPH_FS_SECURITY_LABEL=y +CONFIG_CIFS=m +# CONFIG_CIFS_DEBUG is not set +CONFIG_9P_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_UTF8=y +CONFIG_DLM=m +CONFIG_KEY_DH_OPERATIONS=y +CONFIG_SECURITY=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_SELINUX_BOOTPARAM=y +CONFIG_SECURITY_SELINUX_DISABLE=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_SECURITY_YAMA=y +CONFIG_DEFAULT_SECURITY_DAC=y +CONFIG_CRYPTO_USER=m +# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_PCRYPT=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_CRC32_LOONGARCH=m +CONFIG_CRYPTO_TGR192=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_SALSA20=m +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_DEFLATE=m +CONFIG_CRYPTO_LZO=m +CONFIG_CRYPTO_842=m +CONFIG_CRYPTO_LZ4=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_DEV_VIRTIO=m +CONFIG_DMA_CMA=y +CONFIG_DMA_PERNUMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=0 +CONFIG_PRINTK_TIME=y +CONFIG_STRIP_ASM_SYMS=y +CONFIG_MAGIC_SYSRQ=y +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +# CONFIG_DEBUG_PREEMPT is not set +# CONFIG_FTRACE is not set +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="e1000e.InterruptThrottleRate=4,4,4,4" diff --git a/arch/loongarch/crypto/Makefile b/arch/loongarch/crypto/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d22613d27ce9e00541144dc058c70e858fe71ea3 --- /dev/null +++ b/arch/loongarch/crypto/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for LoongArch crypto files.. +# + +obj-$(CONFIG_CRYPTO_CRC32_LOONGARCH) += crc32-loongarch.o diff --git a/arch/loongarch/crypto/crc32-loongarch.c b/arch/loongarch/crypto/crc32-loongarch.c new file mode 100644 index 0000000000000000000000000000000000000000..10a45faf2ef01352c6c35a72cb1d4cdde64f71d0 --- /dev/null +++ b/arch/loongarch/crypto/crc32-loongarch.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * crc32.c - CRC32 and CRC32C using LoongArch crc* instructions + * + * Module based on mips/crypto/crc32-mips.c + * + * Copyright (C) 2014 Linaro Ltd + * Copyright (C) 2018 MIPS Tech, LLC + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited + */ + +#include +#include +#include + +#include + +#define _CRC32(crc, value, size, type) \ +do { \ + __asm__ __volatile__( \ + #type ".w." #size ".w" " %0, %1, %0\n\t"\ + : "+r" (crc) \ + : "r" (value) \ + : "memory"); \ +} while (0) + +#define CRC32(crc, value, size) _CRC32(crc, value, size, crc) +#define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc) + +static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len) +{ + u32 crc = crc_; + + while (len >= sizeof(u64)) { + u64 value = get_unaligned_le64(p); + + CRC32(crc, value, d); + p += sizeof(u64); + len -= sizeof(u64); + } + + if (len & sizeof(u32)) { + u32 value = get_unaligned_le32(p); + + CRC32(crc, value, w); + p += sizeof(u32); + len -= sizeof(u32); + } + + if (len & sizeof(u16)) { + u16 value = get_unaligned_le16(p); + + CRC32(crc, value, h); + p += sizeof(u16); + } + + if (len & sizeof(u8)) { + u8 value = *p++; + + CRC32(crc, value, b); + } + + return crc; +} + +static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len) +{ + u32 crc = crc_; + + while (len >= sizeof(u64)) { + u64 value = get_unaligned_le64(p); + + CRC32C(crc, value, d); + p += sizeof(u64); + len -= sizeof(u64); + } + + if (len & sizeof(u32)) { + u32 value = get_unaligned_le32(p); + + CRC32C(crc, value, w); + p += sizeof(u32); + len -= sizeof(u32); + } + + if (len & sizeof(u16)) { + u16 value = get_unaligned_le16(p); + + CRC32C(crc, value, h); + p += sizeof(u16); + } + + if (len & sizeof(u8)) { + u8 value = *p++; + + CRC32C(crc, value, b); + } + + return crc; +} + +#define CHKSUM_BLOCK_SIZE 1 +#define CHKSUM_DIGEST_SIZE 4 + +struct chksum_ctx { + u32 key; +}; + +struct chksum_desc_ctx { + u32 crc; +}; + +static int chksum_init(struct shash_desc *desc) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = mctx->key; + + return 0; +} + +/* + * Setting the seed allows arbitrary accumulators and flexible XOR policy + * If your algorithm starts with ~0, then XOR with ~0 before you set the seed. + */ +static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(mctx->key)) + return -EINVAL; + + mctx->key = get_unaligned_le32(key); + + return 0; +} + +static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = crc32_loongarch_hw(ctx->crc, data, length); + return 0; +} + +static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = crc32c_loongarch_hw(ctx->crc, data, length); + return 0; +} + +static int chksum_final(struct shash_desc *desc, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + put_unaligned_le32(ctx->crc, out); + return 0; +} + +static int chksumc_final(struct shash_desc *desc, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + put_unaligned_le32(~ctx->crc, out); + return 0; +} + +static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) +{ + put_unaligned_le32(crc32_loongarch_hw(crc, data, len), out); + return 0; +} + +static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) +{ + put_unaligned_le32(~crc32c_loongarch_hw(crc, data, len), out); + return 0; +} + +static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + return __chksum_finup(ctx->crc, data, len, out); +} + +static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + return __chksumc_finup(ctx->crc, data, len, out); +} + +static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + + return __chksum_finup(mctx->key, data, length, out); +} + +static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + + return __chksumc_finup(mctx->key, data, length, out); +} + +static int chksum_cra_init(struct crypto_tfm *tfm) +{ + struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = 0; + return 0; +} + +static int chksumc_cra_init(struct crypto_tfm *tfm) +{ + struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = ~0; + return 0; +} + +static struct shash_alg crc32_alg = { + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksum_update, + .final = chksum_final, + .finup = chksum_finup, + .digest = chksum_digest, + .descsize = sizeof(struct chksum_desc_ctx), + .base = { + .cra_name = "crc32", + .cra_driver_name = "crc32-loongarch", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 0, + .cra_ctxsize = sizeof(struct chksum_ctx), + .cra_module = THIS_MODULE, + .cra_init = chksum_cra_init, + } +}; + +static struct shash_alg crc32c_alg = { + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksumc_update, + .final = chksumc_final, + .finup = chksumc_finup, + .digest = chksumc_digest, + .descsize = sizeof(struct chksum_desc_ctx), + .base = { + .cra_name = "crc32c", + .cra_driver_name = "crc32c-loongarch", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 0, + .cra_ctxsize = sizeof(struct chksum_ctx), + .cra_module = THIS_MODULE, + .cra_init = chksumc_cra_init, + } +}; + +static int __init crc32_mod_init(void) +{ + int err; + + if (!cpu_has(CPU_FEATURE_CRC32)) + return 0; + + err = crypto_register_shash(&crc32_alg); + if (err) + return err; + + err = crypto_register_shash(&crc32c_alg); + if (err) + return err; + + return 0; +} + +static void __exit crc32_mod_exit(void) +{ + if (!cpu_has(CPU_FEATURE_CRC32)) + return; + + crypto_unregister_shash(&crc32_alg); + crypto_unregister_shash(&crc32c_alg); +} + +module_init(crc32_mod_init); +module_exit(crc32_mod_exit); + +MODULE_AUTHOR("Min Zhou "); +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions"); +MODULE_LICENSE("GPL v2"); diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild new file mode 100644 index 0000000000000000000000000000000000000000..5981af719607e5e2e5785dee0051b848f763f269 --- /dev/null +++ b/arch/loongarch/include/asm/Kbuild @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +generic-y += dma-contiguous.h +generic-y += export.h +generic-y += mcs_spinlock.h +generic-y += parport.h +generic-y += early_ioremap.h +generic-y += qrwlock.h +generic-y += rwsem.h +generic-y += segment.h +generic-y += user.h +generic-y += stat.h +generic-y += fcntl.h +generic-y += ioctl.h +generic-y += ioctls.h +generic-y += mman.h +generic-y += msgbuf.h +generic-y += sembuf.h +generic-y += shmbuf.h +generic-y += statfs.h +generic-y += socket.h +generic-y += sockios.h +generic-y += termios.h +generic-y += termbits.h +generic-y += poll.h +generic-y += param.h +generic-y += posix_types.h +generic-y += resource.h +generic-y += seccomp.h diff --git a/arch/loongarch/include/asm/acenv.h b/arch/loongarch/include/asm/acenv.h new file mode 100644 index 0000000000000000000000000000000000000000..a0353eb2b8c468da31c341a7c48e45c9e346b012 --- /dev/null +++ b/arch/loongarch/include/asm/acenv.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * LoongArch specific ACPICA environments and implementation + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: lvjianmin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_LOONGARCH_ACENV_H +#define _ASM_LOONGARCH_ACENV_H + +/* + * This header is required by ACPI core, but we have nothing to fill in + * right now. Will be updated later when needed. + */ + +#endif /* _ASM_LOONGARCH_ACENV_H */ diff --git a/arch/loongarch/include/asm/acpi.h b/arch/loongarch/include/asm/acpi.h new file mode 100644 index 0000000000000000000000000000000000000000..e76f5bd9b7f3728701304b0786d2039e32fb7c0c --- /dev/null +++ b/arch/loongarch/include/asm/acpi.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Jianmin Lv + * Huacai Chen + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_LOONGARCH_ACPI_H +#define _ASM_LOONGARCH_ACPI_H + +#ifdef CONFIG_ACPI +extern int acpi_strict; +extern int acpi_disabled; +extern int acpi_pci_disabled; +extern int acpi_noirq; +extern int pptt_enabled; + +#define acpi_os_ioremap acpi_os_ioremap +void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size); + +static inline void disable_acpi(void) +{ + acpi_disabled = 1; + acpi_pci_disabled = 1; + acpi_noirq = 1; +} + +static inline bool acpi_has_cpu_in_madt(void) +{ + return true; +} + +#define MAX_CORE_PIC 256 + +extern struct list_head acpi_wakeup_device_list; +extern struct acpi_madt_core_pic acpi_core_pic[MAX_CORE_PIC]; + +extern int __init parse_acpi_topology(void); + +static inline u32 get_acpi_id_for_cpu(unsigned int cpu) +{ + return acpi_core_pic[cpu_logical_map(cpu)].processor_id; +} + +#endif /* !CONFIG_ACPI */ + +#define ACPI_TABLE_UPGRADE_MAX_PHYS ARCH_LOW_ADDRESS_LIMIT + +extern int loongarch_acpi_suspend(void); +extern int (*acpi_suspend_lowlevel)(void); +extern void loongarch_suspend_enter(void); + +static inline unsigned long acpi_get_wakeup_address(void) +{ +#ifdef CONFIG_SUSPEND + extern void loongarch_wakeup_start(void); + return (unsigned long)loongarch_wakeup_start; +#endif + return 0UL; +} + +#endif /* _ASM_LOONGARCH_ACPI_H */ diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h new file mode 100644 index 0000000000000000000000000000000000000000..78d620804f72345a98bd6ac23a567e0d7ff8023a --- /dev/null +++ b/arch/loongarch/include/asm/addrspace.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_ADDRSPACE_H +#define _ASM_ADDRSPACE_H + +#include + +#include + +/* + * This gives the physical RAM offset. + */ +#ifndef __ASSEMBLY__ +#ifndef PHYS_OFFSET +#define PHYS_OFFSET _UL(0) +#endif +extern unsigned long vm_map_base; +#endif /* __ASSEMBLY__ */ + +#ifndef IO_BASE +#define IO_BASE CSR_DMW0_BASE +#endif + +#ifndef CACHE_BASE +#define CACHE_BASE CSR_DMW1_BASE +#endif + +#ifndef UNCACHE_BASE +#define UNCACHE_BASE CSR_DMW0_BASE +#endif + +#define DMW_PABITS 48 +#define TO_PHYS_MASK ((1ULL << DMW_PABITS) - 1) + +/* + * Memory above this physical address will be considered highmem. + */ +#ifndef HIGHMEM_START +#define HIGHMEM_START (_UL(1) << _UL(DMW_PABITS)) +#endif + +#define TO_PHYS(x) ( ((x) & TO_PHYS_MASK)) +#define TO_CACHE(x) (CACHE_BASE | ((x) & TO_PHYS_MASK)) +#define TO_UNCACHE(x) (UNCACHE_BASE | ((x) & TO_PHYS_MASK)) + +/* + * This handles the memory map. + */ +#ifndef PAGE_OFFSET +#define PAGE_OFFSET (CACHE_BASE + PHYS_OFFSET) +#endif + +#ifndef FIXADDR_TOP +#define FIXADDR_TOP ((unsigned long)(long)(int)0xfffe0000) +#endif + +#ifdef __ASSEMBLY__ +#define _ATYPE_ +#define _ATYPE32_ +#define _ATYPE64_ +#else +#define _ATYPE_ __PTRDIFF_TYPE__ +#define _ATYPE32_ int +#define _ATYPE64_ __s64 +#endif + +#ifdef CONFIG_64BIT +#define _CONST64_(x) _UL(x) +#else +#define _CONST64_(x) _ULL(x) +#endif + +/* + * 32/64-bit LoongArch address spaces + */ +#ifdef __ASSEMBLY__ +#define _ACAST32_ +#define _ACAST64_ +#else +#define _ACAST32_ (_ATYPE_)(_ATYPE32_) /* widen if necessary */ +#define _ACAST64_ (_ATYPE64_) /* do _not_ narrow */ +#endif + +#ifdef CONFIG_32BIT + +#define UVRANGE 0x00000000 +#define KPRANGE0 0x80000000 +#define KPRANGE1 0xa0000000 +#define KVRANGE 0xc0000000 + +#else + +#define XUVRANGE _CONST64_(0x0000000000000000) +#define XSPRANGE _CONST64_(0x4000000000000000) +#define XKPRANGE _CONST64_(0x8000000000000000) +#define XKVRANGE _CONST64_(0xc000000000000000) + +#endif + +/* + * Returns the physical address of a KPRANGEx / XKPRANGE address + */ +#define PHYSADDR(a) ((_ACAST64_(a)) & TO_PHYS_MASK) + +/* + * On LoongArch, I/O ports mappring is following: + * + * | .... | + * |-----------------------| + * | pci io ports(16K~32M) | + * |-----------------------| + * | isa io ports(0 ~16K) | + * PCI_IOBASE ->|-----------------------| + * | .... | + */ +#define PCI_IOBASE ((void __iomem *)(vm_map_base + (2 * PAGE_SIZE))) +#define PCI_IOSIZE SZ_32M +#define ISA_IOSIZE SZ_16K +#define IO_SPACE_LIMIT (PCI_IOSIZE - 1) + +#endif /* _ASM_ADDRSPACE_H */ diff --git a/arch/loongarch/include/asm/alternative-asm.h b/arch/loongarch/include/asm/alternative-asm.h new file mode 100644 index 0000000000000000000000000000000000000000..ff3d10ac393f2048ce3e9dddda57ecd2b217a3df --- /dev/null +++ b/arch/loongarch/include/asm/alternative-asm.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ALTERNATIVE_ASM_H +#define _ASM_ALTERNATIVE_ASM_H + +#ifdef __ASSEMBLY__ + +#include + +/* + * Issue one struct alt_instr descriptor entry (need to put it into + * the section .altinstructions, see below). This entry contains + * enough information for the alternatives patching code to patch an + * instruction. See apply_alternatives(). + */ +.macro altinstruction_entry orig alt feature orig_len alt_len + .long \orig - . + .long \alt - . + .short \feature + .byte \orig_len + .byte \alt_len +.endm + +/* + * Define an alternative between two instructions. If @feature is + * present, early code in apply_alternatives() replaces @oldinstr with + * @newinstr. ".fill" directive takes care of proper instruction padding + * in case @newinstr is longer than @oldinstr. + */ +.macro ALTERNATIVE oldinstr, newinstr, feature +140 : + \oldinstr +141 : + .fill - (((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)) / 4, 4, 0x03400000 +142 : + + .pushsection .altinstructions, "a" + altinstruction_entry 140b, 143f, \feature, 142b-140b, 144f-143f + .popsection + + .subsection 1 +143 : + \newinstr +144 : + .previous +.endm + +#define old_len (141b-140b) +#define new_len1 (144f-143f) +#define new_len2 (145f-144f) + +#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b))))) + +/* + * Same as ALTERNATIVE macro above but for two alternatives. If CPU + * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has + * @feature2, it replaces @oldinstr with @feature2. + */ +.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2 +140 : + \oldinstr +141 : + .fill - ((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \ + (alt_max_short(new_len1, new_len2) - (old_len)) / 4, 4, 0x03400000 +142 : + + .pushsection .altinstructions, "a" + altinstruction_entry 140b, 143f, \feature1, 142b-140b, 144f-143f, 142b-141b + altinstruction_entry 140b, 144f, \feature2, 142b-140b, 145f-144f, 142b-141b + .popsection + + .subsection 1 +143 : + \newinstr1 +144 : + \newinstr2 +145 : + .previous +.endm + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/loongarch/include/asm/alternative.h b/arch/loongarch/include/asm/alternative.h new file mode 100644 index 0000000000000000000000000000000000000000..cee7b29785ab250a0a9e5994037b0b0b3c794c1f --- /dev/null +++ b/arch/loongarch/include/asm/alternative.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ALTERNATIVE_H +#define _ASM_ALTERNATIVE_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include + +struct alt_instr { + s32 instr_offset; /* offset to original instruction */ + s32 replace_offset; /* offset to replacement instruction */ + u16 feature; /* feature bit set for replacement */ + u8 instrlen; /* length of original instruction */ + u8 replacementlen; /* length of new instruction */ +} __packed; + +/* + * Debug flag that can be tested to see whether alternative + * instructions were patched in already: + */ +extern int alternatives_patched; +extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; + +extern void alternative_instructions(void); +extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); + +#define b_replacement(num) "664"#num +#define e_replacement(num) "665"#num + +#define alt_end_marker "663" +#define alt_slen "662b-661b" +#define alt_total_slen alt_end_marker"b-661b" +#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f" + +#define __OLDINSTR(oldinstr, num) \ + "661:\n\t" oldinstr "\n662:\n" \ + ".fill -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \ + "((" alt_rlen(num) ")-(" alt_slen ")) / 4, 4, 0x03400000\n" + +#define OLDINSTR(oldinstr, num) \ + __OLDINSTR(oldinstr, num) \ + alt_end_marker ":\n" + +#define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))" + +/* + * Pad the second replacement alternative with additional NOPs if it is + * additionally longer than the first replacement alternative. + */ +#define OLDINSTR_2(oldinstr, num1, num2) \ + "661:\n\t" oldinstr "\n662:\n" \ + ".fill -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \ + "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) / 4, " \ + "4, 0x03400000\n" \ + alt_end_marker ":\n" + +#define ALTINSTR_ENTRY(feature, num) \ + " .long 661b - .\n" /* label */ \ + " .long " b_replacement(num)"f - .\n" /* new instruction */ \ + " .short " __stringify(feature) "\n" /* feature bit */ \ + " .byte " alt_total_slen "\n" /* source len */ \ + " .byte " alt_rlen(num) "\n" /* replacement len */ + +#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \ + b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t" + +/* alternative assembly primitive: */ +#define ALTERNATIVE(oldinstr, newinstr, feature) \ + OLDINSTR(oldinstr, 1) \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feature, 1) \ + ".popsection\n" \ + ".subsection 1\n" \ + ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ + ".previous\n" + +#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ + OLDINSTR_2(oldinstr, 1, 2) \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feature1, 1) \ + ALTINSTR_ENTRY(feature2, 2) \ + ".popsection\n" \ + ".subsection 1\n" \ + ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ + ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ + ".previous\n" + +/* + * Alternative instructions for different CPU types or capabilities. + * + * This allows to use optimized instructions even on generic binary + * kernels. + * + * length of oldinstr must be longer or equal the length of newinstr + * It can be padded with nops as needed. + * + * For non barrier like inlines please define new variants + * without volatile and memory clobber. + */ +#define alternative(oldinstr, newinstr, feature) \ + (asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")) + +#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \ + (asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")) + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ALTERNATIVE_H */ diff --git a/arch/loongarch/include/asm/asm-extable.h b/arch/loongarch/include/asm/asm-extable.h new file mode 100644 index 0000000000000000000000000000000000000000..f5502cb50c6efc8952489ced77dcedcc0a4e1a4d --- /dev/null +++ b/arch/loongarch/include/asm/asm-extable.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_ASM_EXTABLE_H +#define __ASM_ASM_EXTABLE_H + +#define EX_TYPE_NONE 0 +#define EX_TYPE_FIXUP 1 +#define EX_TYPE_UACCESS_ERR_ZERO 2 + +#ifdef __ASSEMBLY__ + +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ + .pushsection __ex_table, "a"; \ + .balign 4; \ + .long ((insn) - .); \ + .long ((fixup) - .); \ + .short (type); \ + .short (data); \ + .popsection; + + .macro _asm_extable, insn, fixup + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0) + .endm + +#else /* __ASSEMBLY__ */ + +#include +#include +#include + +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ + ".pushsection __ex_table, \"a\"\n" \ + ".balign 4\n" \ + ".long ((" insn ") - .)\n" \ + ".long ((" fixup ") - .)\n" \ + ".short (" type ")\n" \ + ".short (" data ")\n" \ + ".popsection\n" + +#define _ASM_EXTABLE(insn, fixup) \ + __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0") + +#define EX_DATA_REG_ERR_SHIFT 0 +#define EX_DATA_REG_ERR GENMASK(4, 0) +#define EX_DATA_REG_ZERO_SHIFT 5 +#define EX_DATA_REG_ZERO GENMASK(9, 5) + +#define EX_DATA_REG(reg, gpr) \ + "((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")" + +#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) \ + __DEFINE_ASM_GPR_NUMS \ + __ASM_EXTABLE_RAW(#insn, #fixup, \ + __stringify(EX_TYPE_UACCESS_ERR_ZERO), \ + "(" \ + EX_DATA_REG(ERR, err) " | " \ + EX_DATA_REG(ZERO, zero) \ + ")") + +#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_ASM_EXTABLE_H */ diff --git a/arch/loongarch/include/asm/asm-offsets.h b/arch/loongarch/include/asm/asm-offsets.h new file mode 100644 index 0000000000000000000000000000000000000000..b49215b280fe7ce83782324b8d49d83075e9c7b3 --- /dev/null +++ b/arch/loongarch/include/asm/asm-offsets.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include diff --git a/arch/loongarch/include/asm/asm-prototypes.h b/arch/loongarch/include/asm/asm-prototypes.h new file mode 100644 index 0000000000000000000000000000000000000000..a63d09f8ee86bd9218ce3c0c3b00b9310826b1bf --- /dev/null +++ b/arch/loongarch/include/asm/asm-prototypes.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include +#include diff --git a/arch/loongarch/include/asm/asm.h b/arch/loongarch/include/asm/asm.h new file mode 100644 index 0000000000000000000000000000000000000000..290c1cc0c8c9514994e0b08632a7f5449c4bda82 --- /dev/null +++ b/arch/loongarch/include/asm/asm.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Some useful macros for LoongArch assembler code + * + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 1995, 1996, 1997, 1999, 2001 by Ralf Baechle + * Copyright (C) 1999 by Silicon Graphics, Inc. + * Copyright (C) 2001 MIPS Technologies, Inc. + * Copyright (C) 2002 Maciej W. Rozycki + */ +#ifndef __ASM_ASM_H +#define __ASM_ASM_H + +/* LoongArch pref instruction. */ +#ifdef CONFIG_CPU_HAS_PREFETCH + +#define PREF(hint, addr, offs) \ + preld hint, addr, offs; \ + +#define PREFX(hint, addr, index) \ + preldx hint, addr, index; \ + +#else /* !CONFIG_CPU_HAS_PREFETCH */ + +#define PREF(hint, addr, offs) +#define PREFX(hint, addr, index) + +#endif /* !CONFIG_CPU_HAS_PREFETCH */ + +/* + * Stack alignment + */ +#define STACK_ALIGN ~(0xf) + +/* + * Macros to handle different pointer/register sizes for 32/64-bit code + */ + +/* + * Size of a register + */ +#ifndef __loongarch64 +#define SZREG 4 +#else +#define SZREG 8 +#endif + +/* + * Use the following macros in assemblercode to load/store registers, + * pointers etc. + */ +#if (SZREG == 4) +#define REG_L ld.w +#define REG_S st.w +#define REG_ADD add.w +#define REG_SUB sub.w +#else /* SZREG == 8 */ +#define REG_L ld.d +#define REG_S st.d +#define REG_ADD add.d +#define REG_SUB sub.d +#endif + +/* + * How to add/sub/load/store/shift C int variables. + */ +#if (__SIZEOF_INT__ == 4) +#define INT_ADD add.w +#define INT_ADDI addi.w +#define INT_SUB sub.w +#define INT_L ld.w +#define INT_S st.w +#define INT_SLL slli.w +#define INT_SLLV sll.w +#define INT_SRL srli.w +#define INT_SRLV srl.w +#define INT_SRA srai.w +#define INT_SRAV sra.w +#endif + +#if (__SIZEOF_INT__ == 8) +#define INT_ADD add.d +#define INT_ADDI addi.d +#define INT_SUB sub.d +#define INT_L ld.d +#define INT_S st.d +#define INT_SLL slli.d +#define INT_SLLV sll.d +#define INT_SRL srli.d +#define INT_SRLV srl.d +#define INT_SRA srai.d +#define INT_SRAV sra.d +#endif + +/* + * How to add/sub/load/store/shift C long variables. + */ +#if (__SIZEOF_LONG__ == 4) +#define LONG_ADD add.w +#define LONG_ADDI addi.w +#define LONG_SUB sub.w +#define LONG_L ld.w +#define LONG_S st.w +#define LONG_SLL slli.w +#define LONG_SLLV sll.w +#define LONG_SRL srli.w +#define LONG_SRLV srl.w +#define LONG_SRA srai.w +#define LONG_SRAV sra.w + +#ifdef __ASSEMBLY__ +#define LONG .word +#endif +#define LONGSIZE 4 +#define LONGMASK 3 +#define LONGLOG 2 +#endif + +#if (__SIZEOF_LONG__ == 8) +#define LONG_ADD add.d +#define LONG_ADDI addi.d +#define LONG_SUB sub.d +#define LONG_L ld.d +#define LONG_S st.d +#define LONG_SLL slli.d +#define LONG_SLLV sll.d +#define LONG_SRL srli.d +#define LONG_SRLV srl.d +#define LONG_SRA srai.d +#define LONG_SRAV sra.d + +#ifdef __ASSEMBLY__ +#define LONG .dword +#endif +#define LONGSIZE 8 +#define LONGMASK 7 +#define LONGLOG 3 +#endif + +/* + * How to add/sub/load/store/shift pointers. + */ +#if (__SIZEOF_POINTER__ == 4) +#define PTR_ADD add.w +#define PTR_ADDI addi.w +#define PTR_SUB sub.w +#define PTR_L ld.w +#define PTR_S st.w +#define PTR_LI li.w +#define PTR_SLL slli.w +#define PTR_SLLV sll.w +#define PTR_SRL srli.w +#define PTR_SRLV srl.w +#define PTR_SRA srai.w +#define PTR_SRAV sra.w + +#define PTR_SCALESHIFT 2 + +#ifdef __ASSEMBLY__ +#define PTR .word +#endif +#define PTRSIZE 4 +#define PTRLOG 2 +#endif + +#if (__SIZEOF_POINTER__ == 8) +#define PTR_ADD add.d +#define PTR_ADDI addi.d +#define PTR_SUB sub.d +#define PTR_L ld.d +#define PTR_S st.d +#define PTR_LI li.d +#define PTR_SLL slli.d +#define PTR_SLLV sll.d +#define PTR_SRL srli.d +#define PTR_SRLV srl.d +#define PTR_SRA srai.d +#define PTR_SRAV sra.d + +#define PTR_SCALESHIFT 3 + +#ifdef __ASSEMBLY__ +#define PTR .dword +#endif +#define PTRSIZE 8 +#define PTRLOG 3 +#endif + +#endif /* __ASM_ASM_H */ diff --git a/arch/loongarch/include/asm/asmmacro-64.h b/arch/loongarch/include/asm/asmmacro-64.h new file mode 100644 index 0000000000000000000000000000000000000000..90843d1002d955c6f5fa413390df7bcb961f2e79 --- /dev/null +++ b/arch/loongarch/include/asm/asmmacro-64.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * asmmacro.h: Assembler macros to make things easier to read. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_ASMMACRO_64_H +#define _ASM_ASMMACRO_64_H + +#include +#include +#include +#include + + .macro cpu_save_nonscratch thread + stptr.d s0, \thread, THREAD_REG23 + stptr.d s1, \thread, THREAD_REG24 + stptr.d s2, \thread, THREAD_REG25 + stptr.d s3, \thread, THREAD_REG26 + stptr.d s4, \thread, THREAD_REG27 + stptr.d s5, \thread, THREAD_REG28 + stptr.d s6, \thread, THREAD_REG29 + stptr.d s7, \thread, THREAD_REG30 + stptr.d s8, \thread, THREAD_REG31 + stptr.d sp, \thread, THREAD_REG03 + stptr.d fp, \thread, THREAD_REG22 + .endm + + .macro cpu_restore_nonscratch thread + ldptr.d s0, \thread, THREAD_REG23 + ldptr.d s1, \thread, THREAD_REG24 + ldptr.d s2, \thread, THREAD_REG25 + ldptr.d s3, \thread, THREAD_REG26 + ldptr.d s4, \thread, THREAD_REG27 + ldptr.d s5, \thread, THREAD_REG28 + ldptr.d s6, \thread, THREAD_REG29 + ldptr.d s7, \thread, THREAD_REG30 + ldptr.d s8, \thread, THREAD_REG31 + ldptr.d sp, \thread, THREAD_REG03 + ldptr.d fp, \thread, THREAD_REG22 + ldptr.d ra, \thread, THREAD_REG01 + .endm + +#endif /* _ASM_ASMMACRO_64_H */ diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h new file mode 100644 index 0000000000000000000000000000000000000000..952c770afca572cf597d778a4325f37a2348c55a --- /dev/null +++ b/arch/loongarch/include/asm/asmmacro.h @@ -0,0 +1,935 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_ASMMACRO_H +#define _ASM_ASMMACRO_H + +#include + +#ifdef CONFIG_32BIT +#include +#endif +#ifdef CONFIG_64BIT +#include +#endif + + .macro parse_v var val + \var = \val + .endm + + .macro parse_r var r + \var = -1 + .ifc \r, $r0 + \var = 0 + .endif + .ifc \r, $r1 + \var = 1 + .endif + .ifc \r, $r2 + \var = 2 + .endif + .ifc \r, $r3 + \var = 3 + .endif + .ifc \r, $r4 + \var = 4 + .endif + .ifc \r, $r5 + \var = 5 + .endif + .ifc \r, $r6 + \var = 6 + .endif + .ifc \r, $r7 + \var = 7 + .endif + .ifc \r, $r8 + \var = 8 + .endif + .ifc \r, $r9 + \var = 9 + .endif + .ifc \r, $r10 + \var = 10 + .endif + .ifc \r, $r11 + \var = 11 + .endif + .ifc \r, $r12 + \var = 12 + .endif + .ifc \r, $r13 + \var = 13 + .endif + .ifc \r, $r14 + \var = 14 + .endif + .ifc \r, $r15 + \var = 15 + .endif + .ifc \r, $r16 + \var = 16 + .endif + .ifc \r, $r17 + \var = 17 + .endif + .ifc \r, $r18 + \var = 18 + .endif + .ifc \r, $r19 + \var = 19 + .endif + .ifc \r, $r20 + \var = 20 + .endif + .ifc \r, $r21 + \var = 21 + .endif + .ifc \r, $r22 + \var = 22 + .endif + .ifc \r, $r23 + \var = 23 + .endif + .ifc \r, $r24 + \var = 24 + .endif + .ifc \r, $r25 + \var = 25 + .endif + .ifc \r, $r26 + \var = 26 + .endif + .ifc \r, $r27 + \var = 27 + .endif + .ifc \r, $r28 + \var = 28 + .endif + .ifc \r, $r29 + \var = 29 + .endif + .ifc \r, $r30 + \var = 30 + .endif + .ifc \r, $r31 + \var = 31 + .endif + .iflt \var + .error "Unable to parse register name \r" + .endif + .endm + + .macro parse_vr var vr + \var = -1 + .ifc \vr, $vr0 + \var = 0 + .endif + .ifc \vr, $vr1 + \var = 1 + .endif + .ifc \vr, $vr2 + \var = 2 + .endif + .ifc \vr, $vr3 + \var = 3 + .endif + .ifc \vr, $vr4 + \var = 4 + .endif + .ifc \vr, $vr5 + \var = 5 + .endif + .ifc \vr, $vr6 + \var = 6 + .endif + .ifc \vr, $vr7 + \var = 7 + .endif + .ifc \vr, $vr8 + \var = 8 + .endif + .ifc \vr, $vr9 + \var = 9 + .endif + .ifc \vr, $vr10 + \var = 10 + .endif + .ifc \vr, $vr11 + \var = 11 + .endif + .ifc \vr, $vr12 + \var = 12 + .endif + .ifc \vr, $vr13 + \var = 13 + .endif + .ifc \vr, $vr14 + \var = 14 + .endif + .ifc \vr, $vr15 + \var = 15 + .endif + .ifc \vr, $vr16 + \var = 16 + .endif + .ifc \vr, $vr17 + \var = 17 + .endif + .ifc \vr, $vr18 + \var = 18 + .endif + .ifc \vr, $vr19 + \var = 19 + .endif + .ifc \vr, $vr20 + \var = 20 + .endif + .ifc \vr, $vr21 + \var = 21 + .endif + .ifc \vr, $vr22 + \var = 22 + .endif + .ifc \vr, $vr23 + \var = 23 + .endif + .ifc \vr, $vr24 + \var = 24 + .endif + .ifc \vr, $vr25 + \var = 25 + .endif + .ifc \vr, $vr26 + \var = 26 + .endif + .ifc \vr, $vr27 + \var = 27 + .endif + .ifc \vr, $vr28 + \var = 28 + .endif + .ifc \vr, $vr29 + \var = 29 + .endif + .ifc \vr, $vr30 + \var = 30 + .endif + .ifc \vr, $vr31 + \var = 31 + .endif + .iflt \var + .error "Unable to parse register name \r" + .endif + .endm + + .macro parse_xr var xr + \var = -1 + .ifc \xr, $xr0 + \var = 0 + .endif + .ifc \xr, $xr1 + \var = 1 + .endif + .ifc \xr, $xr2 + \var = 2 + .endif + .ifc \xr, $xr3 + \var = 3 + .endif + .ifc \xr, $xr4 + \var = 4 + .endif + .ifc \xr, $xr5 + \var = 5 + .endif + .ifc \xr, $xr6 + \var = 6 + .endif + .ifc \xr, $xr7 + \var = 7 + .endif + .ifc \xr, $xr8 + \var = 8 + .endif + .ifc \xr, $xr9 + \var = 9 + .endif + .ifc \xr, $xr10 + \var = 10 + .endif + .ifc \xr, $xr11 + \var = 11 + .endif + .ifc \xr, $xr12 + \var = 12 + .endif + .ifc \xr, $xr13 + \var = 13 + .endif + .ifc \xr, $xr14 + \var = 14 + .endif + .ifc \xr, $xr15 + \var = 15 + .endif + .ifc \xr, $xr16 + \var = 16 + .endif + .ifc \xr, $xr17 + \var = 17 + .endif + .ifc \xr, $xr18 + \var = 18 + .endif + .ifc \xr, $xr19 + \var = 19 + .endif + .ifc \xr, $xr20 + \var = 20 + .endif + .ifc \xr, $xr21 + \var = 21 + .endif + .ifc \xr, $xr22 + \var = 22 + .endif + .ifc \xr, $xr23 + \var = 23 + .endif + .ifc \xr, $xr24 + \var = 24 + .endif + .ifc \xr, $xr25 + \var = 25 + .endif + .ifc \xr, $xr26 + \var = 26 + .endif + .ifc \xr, $xr27 + \var = 27 + .endif + .ifc \xr, $xr28 + \var = 28 + .endif + .ifc \xr, $xr29 + \var = 29 + .endif + .ifc \xr, $xr30 + \var = 30 + .endif + .ifc \xr, $xr31 + \var = 31 + .endif + .iflt \var + .error "Unable to parse register name \r" + .endif + .endm + + .macro fpu_save_csr thread tmp + movfcsr2gr \tmp, fcsr0 + stptr.w \tmp, \thread, THREAD_FCSR +#ifdef CONFIG_CPU_HAS_LBT + /* TM bit is always 0 if LBT not supported */ + andi \tmp, \tmp, FPU_CSR_TM + beqz \tmp, 1f + bstrins.d \tmp, zero, FPU_CSR_TM_SHIFT, FPU_CSR_TM_SHIFT + movgr2fcsr fcsr0, \tmp + x86mftop \tmp + stptr.d \tmp, \thread, THREAD_FTOP +1: +#endif + .endm + + .macro fpu_restore_csr thread tmp + ldptr.w \tmp, \thread, THREAD_FCSR + movgr2fcsr fcsr0, \tmp +#ifdef CONFIG_CPU_HAS_LBT + /* TM bit is always 0 if LBT not supported */ + andi \tmp, \tmp, FPU_CSR_TM + beqz \tmp, 1f + ldptr.d \tmp, \thread, THREAD_FTOP + x86mttop 0x0 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x1 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x2 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x3 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x4 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x5 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x6 + beq \tmp, zero, 1f + addi.d \tmp, \tmp, -1 + x86mttop 0x7 +1: +#endif + .endm + + .macro fpu_save_cc thread tmp0 tmp1 + movcf2gr \tmp0, $fcc0 + move \tmp1, \tmp0 + movcf2gr \tmp0, $fcc1 + bstrins.d \tmp1, \tmp0, 15, 8 + movcf2gr \tmp0, $fcc2 + bstrins.d \tmp1, \tmp0, 23, 16 + movcf2gr \tmp0, $fcc3 + bstrins.d \tmp1, \tmp0, 31, 24 + movcf2gr \tmp0, $fcc4 + bstrins.d \tmp1, \tmp0, 39, 32 + movcf2gr \tmp0, $fcc5 + bstrins.d \tmp1, \tmp0, 47, 40 + movcf2gr \tmp0, $fcc6 + bstrins.d \tmp1, \tmp0, 55, 48 + movcf2gr \tmp0, $fcc7 + bstrins.d \tmp1, \tmp0, 63, 56 + stptr.d \tmp1, \thread, THREAD_FCC + .endm + + .macro fpu_restore_cc thread tmp0 tmp1 + ldptr.d \tmp0, \thread, THREAD_FCC + bstrpick.d \tmp1, \tmp0, 7, 0 + movgr2cf $fcc0, \tmp1 + bstrpick.d \tmp1, \tmp0, 15, 8 + movgr2cf $fcc1, \tmp1 + bstrpick.d \tmp1, \tmp0, 23, 16 + movgr2cf $fcc2, \tmp1 + bstrpick.d \tmp1, \tmp0, 31, 24 + movgr2cf $fcc3, \tmp1 + bstrpick.d \tmp1, \tmp0, 39, 32 + movgr2cf $fcc4, \tmp1 + bstrpick.d \tmp1, \tmp0, 47, 40 + movgr2cf $fcc5, \tmp1 + bstrpick.d \tmp1, \tmp0, 55, 48 + movgr2cf $fcc6, \tmp1 + bstrpick.d \tmp1, \tmp0, 63, 56 + movgr2cf $fcc7, \tmp1 + .endm + + .macro fpu_save_double thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \tmp, \thread + fst.d $f0, \tmp, THREAD_FPR0 - THREAD_FPR0 + fst.d $f1, \tmp, THREAD_FPR1 - THREAD_FPR0 + fst.d $f2, \tmp, THREAD_FPR2 - THREAD_FPR0 + fst.d $f3, \tmp, THREAD_FPR3 - THREAD_FPR0 + fst.d $f4, \tmp, THREAD_FPR4 - THREAD_FPR0 + fst.d $f5, \tmp, THREAD_FPR5 - THREAD_FPR0 + fst.d $f6, \tmp, THREAD_FPR6 - THREAD_FPR0 + fst.d $f7, \tmp, THREAD_FPR7 - THREAD_FPR0 + fst.d $f8, \tmp, THREAD_FPR8 - THREAD_FPR0 + fst.d $f9, \tmp, THREAD_FPR9 - THREAD_FPR0 + fst.d $f10, \tmp, THREAD_FPR10 - THREAD_FPR0 + fst.d $f11, \tmp, THREAD_FPR11 - THREAD_FPR0 + fst.d $f12, \tmp, THREAD_FPR12 - THREAD_FPR0 + fst.d $f13, \tmp, THREAD_FPR13 - THREAD_FPR0 + fst.d $f14, \tmp, THREAD_FPR14 - THREAD_FPR0 + fst.d $f15, \tmp, THREAD_FPR15 - THREAD_FPR0 + fst.d $f16, \tmp, THREAD_FPR16 - THREAD_FPR0 + fst.d $f17, \tmp, THREAD_FPR17 - THREAD_FPR0 + fst.d $f18, \tmp, THREAD_FPR18 - THREAD_FPR0 + fst.d $f19, \tmp, THREAD_FPR19 - THREAD_FPR0 + fst.d $f20, \tmp, THREAD_FPR20 - THREAD_FPR0 + fst.d $f21, \tmp, THREAD_FPR21 - THREAD_FPR0 + fst.d $f22, \tmp, THREAD_FPR22 - THREAD_FPR0 + fst.d $f23, \tmp, THREAD_FPR23 - THREAD_FPR0 + fst.d $f24, \tmp, THREAD_FPR24 - THREAD_FPR0 + fst.d $f25, \tmp, THREAD_FPR25 - THREAD_FPR0 + fst.d $f26, \tmp, THREAD_FPR26 - THREAD_FPR0 + fst.d $f27, \tmp, THREAD_FPR27 - THREAD_FPR0 + fst.d $f28, \tmp, THREAD_FPR28 - THREAD_FPR0 + fst.d $f29, \tmp, THREAD_FPR29 - THREAD_FPR0 + fst.d $f30, \tmp, THREAD_FPR30 - THREAD_FPR0 + fst.d $f31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro fpu_restore_double thread tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \tmp, \thread + fld.d $f0, \tmp, THREAD_FPR0 - THREAD_FPR0 + fld.d $f1, \tmp, THREAD_FPR1 - THREAD_FPR0 + fld.d $f2, \tmp, THREAD_FPR2 - THREAD_FPR0 + fld.d $f3, \tmp, THREAD_FPR3 - THREAD_FPR0 + fld.d $f4, \tmp, THREAD_FPR4 - THREAD_FPR0 + fld.d $f5, \tmp, THREAD_FPR5 - THREAD_FPR0 + fld.d $f6, \tmp, THREAD_FPR6 - THREAD_FPR0 + fld.d $f7, \tmp, THREAD_FPR7 - THREAD_FPR0 + fld.d $f8, \tmp, THREAD_FPR8 - THREAD_FPR0 + fld.d $f9, \tmp, THREAD_FPR9 - THREAD_FPR0 + fld.d $f10, \tmp, THREAD_FPR10 - THREAD_FPR0 + fld.d $f11, \tmp, THREAD_FPR11 - THREAD_FPR0 + fld.d $f12, \tmp, THREAD_FPR12 - THREAD_FPR0 + fld.d $f13, \tmp, THREAD_FPR13 - THREAD_FPR0 + fld.d $f14, \tmp, THREAD_FPR14 - THREAD_FPR0 + fld.d $f15, \tmp, THREAD_FPR15 - THREAD_FPR0 + fld.d $f16, \tmp, THREAD_FPR16 - THREAD_FPR0 + fld.d $f17, \tmp, THREAD_FPR17 - THREAD_FPR0 + fld.d $f18, \tmp, THREAD_FPR18 - THREAD_FPR0 + fld.d $f19, \tmp, THREAD_FPR19 - THREAD_FPR0 + fld.d $f20, \tmp, THREAD_FPR20 - THREAD_FPR0 + fld.d $f21, \tmp, THREAD_FPR21 - THREAD_FPR0 + fld.d $f22, \tmp, THREAD_FPR22 - THREAD_FPR0 + fld.d $f23, \tmp, THREAD_FPR23 - THREAD_FPR0 + fld.d $f24, \tmp, THREAD_FPR24 - THREAD_FPR0 + fld.d $f25, \tmp, THREAD_FPR25 - THREAD_FPR0 + fld.d $f26, \tmp, THREAD_FPR26 - THREAD_FPR0 + fld.d $f27, \tmp, THREAD_FPR27 - THREAD_FPR0 + fld.d $f28, \tmp, THREAD_FPR28 - THREAD_FPR0 + fld.d $f29, \tmp, THREAD_FPR29 - THREAD_FPR0 + fld.d $f30, \tmp, THREAD_FPR30 - THREAD_FPR0 + fld.d $f31, \tmp, THREAD_FPR31 - THREAD_FPR0 + .endm + + .macro lsx_save_data thread tmp + parse_r __tmp, \tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + /* vst opcode is 0xb1 */ + .word (0xb1 << 22 | ((THREAD_FPR0-THREAD_FPR0) << 10) | __tmp << 5 | 0) + .word (0xb1 << 22 | ((THREAD_FPR1-THREAD_FPR0) << 10) | __tmp << 5 | 1) + .word (0xb1 << 22 | ((THREAD_FPR2-THREAD_FPR0) << 10) | __tmp << 5 | 2) + .word (0xb1 << 22 | ((THREAD_FPR3-THREAD_FPR0) << 10) | __tmp << 5 | 3) + .word (0xb1 << 22 | ((THREAD_FPR4-THREAD_FPR0) << 10) | __tmp << 5 | 4) + .word (0xb1 << 22 | ((THREAD_FPR5-THREAD_FPR0) << 10) | __tmp << 5 | 5) + .word (0xb1 << 22 | ((THREAD_FPR6-THREAD_FPR0) << 10) | __tmp << 5 | 6) + .word (0xb1 << 22 | ((THREAD_FPR7-THREAD_FPR0) << 10) | __tmp << 5 | 7) + .word (0xb1 << 22 | ((THREAD_FPR8-THREAD_FPR0) << 10) | __tmp << 5 | 8) + .word (0xb1 << 22 | ((THREAD_FPR9-THREAD_FPR0) << 10) | __tmp << 5 | 9) + .word (0xb1 << 22 | ((THREAD_FPR10-THREAD_FPR0) << 10) | __tmp << 5 | 10) + .word (0xb1 << 22 | ((THREAD_FPR11-THREAD_FPR0) << 10) | __tmp << 5 | 11) + .word (0xb1 << 22 | ((THREAD_FPR12-THREAD_FPR0) << 10) | __tmp << 5 | 12) + .word (0xb1 << 22 | ((THREAD_FPR13-THREAD_FPR0) << 10) | __tmp << 5 | 13) + .word (0xb1 << 22 | ((THREAD_FPR14-THREAD_FPR0) << 10) | __tmp << 5 | 14) + .word (0xb1 << 22 | ((THREAD_FPR15-THREAD_FPR0) << 10) | __tmp << 5 | 15) + .word (0xb1 << 22 | ((THREAD_FPR16-THREAD_FPR0) << 10) | __tmp << 5 | 16) + .word (0xb1 << 22 | ((THREAD_FPR17-THREAD_FPR0) << 10) | __tmp << 5 | 17) + .word (0xb1 << 22 | ((THREAD_FPR18-THREAD_FPR0) << 10) | __tmp << 5 | 18) + .word (0xb1 << 22 | ((THREAD_FPR19-THREAD_FPR0) << 10) | __tmp << 5 | 19) + .word (0xb1 << 22 | ((THREAD_FPR20-THREAD_FPR0) << 10) | __tmp << 5 | 20) + .word (0xb1 << 22 | ((THREAD_FPR21-THREAD_FPR0) << 10) | __tmp << 5 | 21) + .word (0xb1 << 22 | ((THREAD_FPR22-THREAD_FPR0) << 10) | __tmp << 5 | 22) + .word (0xb1 << 22 | ((THREAD_FPR23-THREAD_FPR0) << 10) | __tmp << 5 | 23) + .word (0xb1 << 22 | ((THREAD_FPR24-THREAD_FPR0) << 10) | __tmp << 5 | 24) + .word (0xb1 << 22 | ((THREAD_FPR25-THREAD_FPR0) << 10) | __tmp << 5 | 25) + .word (0xb1 << 22 | ((THREAD_FPR26-THREAD_FPR0) << 10) | __tmp << 5 | 26) + .word (0xb1 << 22 | ((THREAD_FPR27-THREAD_FPR0) << 10) | __tmp << 5 | 27) + .word (0xb1 << 22 | ((THREAD_FPR28-THREAD_FPR0) << 10) | __tmp << 5 | 28) + .word (0xb1 << 22 | ((THREAD_FPR29-THREAD_FPR0) << 10) | __tmp << 5 | 29) + .word (0xb1 << 22 | ((THREAD_FPR30-THREAD_FPR0) << 10) | __tmp << 5 | 30) + .word (0xb1 << 22 | ((THREAD_FPR31-THREAD_FPR0) << 10) | __tmp << 5 | 31) + .endm + + .macro lsx_restore_data thread tmp + parse_r __tmp, \tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + /* vld opcode is 0xb0 */ + .word (0xb0 << 22 | ((THREAD_FPR0-THREAD_FPR0) << 10) | __tmp << 5 | 0) + .word (0xb0 << 22 | ((THREAD_FPR1-THREAD_FPR0) << 10) | __tmp << 5 | 1) + .word (0xb0 << 22 | ((THREAD_FPR2-THREAD_FPR0) << 10) | __tmp << 5 | 2) + .word (0xb0 << 22 | ((THREAD_FPR3-THREAD_FPR0) << 10) | __tmp << 5 | 3) + .word (0xb0 << 22 | ((THREAD_FPR4-THREAD_FPR0) << 10) | __tmp << 5 | 4) + .word (0xb0 << 22 | ((THREAD_FPR5-THREAD_FPR0) << 10) | __tmp << 5 | 5) + .word (0xb0 << 22 | ((THREAD_FPR6-THREAD_FPR0) << 10) | __tmp << 5 | 6) + .word (0xb0 << 22 | ((THREAD_FPR7-THREAD_FPR0) << 10) | __tmp << 5 | 7) + .word (0xb0 << 22 | ((THREAD_FPR8-THREAD_FPR0) << 10) | __tmp << 5 | 8) + .word (0xb0 << 22 | ((THREAD_FPR9-THREAD_FPR0) << 10) | __tmp << 5 | 9) + .word (0xb0 << 22 | ((THREAD_FPR10-THREAD_FPR0) << 10) | __tmp << 5 | 10) + .word (0xb0 << 22 | ((THREAD_FPR11-THREAD_FPR0) << 10) | __tmp << 5 | 11) + .word (0xb0 << 22 | ((THREAD_FPR12-THREAD_FPR0) << 10) | __tmp << 5 | 12) + .word (0xb0 << 22 | ((THREAD_FPR13-THREAD_FPR0) << 10) | __tmp << 5 | 13) + .word (0xb0 << 22 | ((THREAD_FPR14-THREAD_FPR0) << 10) | __tmp << 5 | 14) + .word (0xb0 << 22 | ((THREAD_FPR15-THREAD_FPR0) << 10) | __tmp << 5 | 15) + .word (0xb0 << 22 | ((THREAD_FPR16-THREAD_FPR0) << 10) | __tmp << 5 | 16) + .word (0xb0 << 22 | ((THREAD_FPR17-THREAD_FPR0) << 10) | __tmp << 5 | 17) + .word (0xb0 << 22 | ((THREAD_FPR18-THREAD_FPR0) << 10) | __tmp << 5 | 18) + .word (0xb0 << 22 | ((THREAD_FPR19-THREAD_FPR0) << 10) | __tmp << 5 | 19) + .word (0xb0 << 22 | ((THREAD_FPR20-THREAD_FPR0) << 10) | __tmp << 5 | 20) + .word (0xb0 << 22 | ((THREAD_FPR21-THREAD_FPR0) << 10) | __tmp << 5 | 21) + .word (0xb0 << 22 | ((THREAD_FPR22-THREAD_FPR0) << 10) | __tmp << 5 | 22) + .word (0xb0 << 22 | ((THREAD_FPR23-THREAD_FPR0) << 10) | __tmp << 5 | 23) + .word (0xb0 << 22 | ((THREAD_FPR24-THREAD_FPR0) << 10) | __tmp << 5 | 24) + .word (0xb0 << 22 | ((THREAD_FPR25-THREAD_FPR0) << 10) | __tmp << 5 | 25) + .word (0xb0 << 22 | ((THREAD_FPR26-THREAD_FPR0) << 10) | __tmp << 5 | 26) + .word (0xb0 << 22 | ((THREAD_FPR27-THREAD_FPR0) << 10) | __tmp << 5 | 27) + .word (0xb0 << 22 | ((THREAD_FPR28-THREAD_FPR0) << 10) | __tmp << 5 | 28) + .word (0xb0 << 22 | ((THREAD_FPR29-THREAD_FPR0) << 10) | __tmp << 5 | 29) + .word (0xb0 << 22 | ((THREAD_FPR30-THREAD_FPR0) << 10) | __tmp << 5 | 30) + .word (0xb0 << 22 | ((THREAD_FPR31-THREAD_FPR0) << 10) | __tmp << 5 | 31) + .endm + + .macro lsx_save_all thread tmp0 tmp1 + fpu_save_cc \thread, \tmp0, \tmp1 + fpu_save_csr \thread, \tmp0 + lsx_save_data \thread, \tmp0 + .endm + + .macro lsx_restore_all thread tmp0 tmp1 + lsx_restore_data \thread, \tmp0 + fpu_restore_cc \thread, \tmp0, \tmp1 + fpu_restore_csr \thread, \tmp0 + .endm + + .macro lsx_save_upper vd base tmp off + parse_vr __vd, \vd + parse_r __tmp, \tmp + /* vpickve2gr opcode is 0xe5dfe */ + .word (0xe5dfe << 11 | 1 << 10 | __vd << 5 | __tmp) + st.d \tmp, \base, (\off+8) + .endm + + .macro lsx_save_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + lsx_save_upper $vr0, \base, \tmp, (THREAD_FPR0-THREAD_FPR0) + lsx_save_upper $vr1, \base, \tmp, (THREAD_FPR1-THREAD_FPR0) + lsx_save_upper $vr2, \base, \tmp, (THREAD_FPR2-THREAD_FPR0) + lsx_save_upper $vr3, \base, \tmp, (THREAD_FPR3-THREAD_FPR0) + lsx_save_upper $vr4, \base, \tmp, (THREAD_FPR4-THREAD_FPR0) + lsx_save_upper $vr5, \base, \tmp, (THREAD_FPR5-THREAD_FPR0) + lsx_save_upper $vr6, \base, \tmp, (THREAD_FPR6-THREAD_FPR0) + lsx_save_upper $vr7, \base, \tmp, (THREAD_FPR7-THREAD_FPR0) + lsx_save_upper $vr8, \base, \tmp, (THREAD_FPR8-THREAD_FPR0) + lsx_save_upper $vr9, \base, \tmp, (THREAD_FPR9-THREAD_FPR0) + lsx_save_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0) + lsx_save_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0) + lsx_save_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0) + lsx_save_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0) + lsx_save_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0) + lsx_save_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0) + lsx_save_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0) + lsx_save_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0) + lsx_save_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0) + lsx_save_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0) + lsx_save_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0) + lsx_save_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0) + lsx_save_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0) + lsx_save_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0) + lsx_save_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0) + lsx_save_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0) + lsx_save_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0) + lsx_save_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0) + lsx_save_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0) + lsx_save_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0) + lsx_save_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0) + lsx_save_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0) + .endm + + .macro lsx_restore_upper vd base tmp off + parse_vr __vd, \vd + parse_r __tmp, \tmp + ld.d \tmp, \base, (\off+8) + /* vinsgr2vr opcode is 0xe5d7e */ + .word (0xe5d7e << 11 | 1 << 10 | __tmp << 5 | __vd) + .endm + + .macro lsx_restore_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + lsx_restore_upper $vr0, \base, \tmp, (THREAD_FPR0-THREAD_FPR0) + lsx_restore_upper $vr1, \base, \tmp, (THREAD_FPR1-THREAD_FPR0) + lsx_restore_upper $vr2, \base, \tmp, (THREAD_FPR2-THREAD_FPR0) + lsx_restore_upper $vr3, \base, \tmp, (THREAD_FPR3-THREAD_FPR0) + lsx_restore_upper $vr4, \base, \tmp, (THREAD_FPR4-THREAD_FPR0) + lsx_restore_upper $vr5, \base, \tmp, (THREAD_FPR5-THREAD_FPR0) + lsx_restore_upper $vr6, \base, \tmp, (THREAD_FPR6-THREAD_FPR0) + lsx_restore_upper $vr7, \base, \tmp, (THREAD_FPR7-THREAD_FPR0) + lsx_restore_upper $vr8, \base, \tmp, (THREAD_FPR8-THREAD_FPR0) + lsx_restore_upper $vr9, \base, \tmp, (THREAD_FPR9-THREAD_FPR0) + lsx_restore_upper $vr10, \base, \tmp, (THREAD_FPR10-THREAD_FPR0) + lsx_restore_upper $vr11, \base, \tmp, (THREAD_FPR11-THREAD_FPR0) + lsx_restore_upper $vr12, \base, \tmp, (THREAD_FPR12-THREAD_FPR0) + lsx_restore_upper $vr13, \base, \tmp, (THREAD_FPR13-THREAD_FPR0) + lsx_restore_upper $vr14, \base, \tmp, (THREAD_FPR14-THREAD_FPR0) + lsx_restore_upper $vr15, \base, \tmp, (THREAD_FPR15-THREAD_FPR0) + lsx_restore_upper $vr16, \base, \tmp, (THREAD_FPR16-THREAD_FPR0) + lsx_restore_upper $vr17, \base, \tmp, (THREAD_FPR17-THREAD_FPR0) + lsx_restore_upper $vr18, \base, \tmp, (THREAD_FPR18-THREAD_FPR0) + lsx_restore_upper $vr19, \base, \tmp, (THREAD_FPR19-THREAD_FPR0) + lsx_restore_upper $vr20, \base, \tmp, (THREAD_FPR20-THREAD_FPR0) + lsx_restore_upper $vr21, \base, \tmp, (THREAD_FPR21-THREAD_FPR0) + lsx_restore_upper $vr22, \base, \tmp, (THREAD_FPR22-THREAD_FPR0) + lsx_restore_upper $vr23, \base, \tmp, (THREAD_FPR23-THREAD_FPR0) + lsx_restore_upper $vr24, \base, \tmp, (THREAD_FPR24-THREAD_FPR0) + lsx_restore_upper $vr25, \base, \tmp, (THREAD_FPR25-THREAD_FPR0) + lsx_restore_upper $vr26, \base, \tmp, (THREAD_FPR26-THREAD_FPR0) + lsx_restore_upper $vr27, \base, \tmp, (THREAD_FPR27-THREAD_FPR0) + lsx_restore_upper $vr28, \base, \tmp, (THREAD_FPR28-THREAD_FPR0) + lsx_restore_upper $vr29, \base, \tmp, (THREAD_FPR29-THREAD_FPR0) + lsx_restore_upper $vr30, \base, \tmp, (THREAD_FPR30-THREAD_FPR0) + lsx_restore_upper $vr31, \base, \tmp, (THREAD_FPR31-THREAD_FPR0) + .endm + + .macro lsx_init_upper vd tmp + parse_vr __vd, \vd + parse_r __tmp, \tmp + /* vinsgr2vr opcode is 0xe5d7e */ + .word (0xe5d7e << 11 | 1 << 10 | __tmp << 5 | __vd) + .endm + + .macro lsx_init_all_upper tmp + not \tmp, zero + lsx_init_upper $vr0 \tmp + lsx_init_upper $vr1 \tmp + lsx_init_upper $vr2 \tmp + lsx_init_upper $vr3 \tmp + lsx_init_upper $vr4 \tmp + lsx_init_upper $vr5 \tmp + lsx_init_upper $vr6 \tmp + lsx_init_upper $vr7 \tmp + lsx_init_upper $vr8 \tmp + lsx_init_upper $vr9 \tmp + lsx_init_upper $vr10 \tmp + lsx_init_upper $vr11 \tmp + lsx_init_upper $vr12 \tmp + lsx_init_upper $vr13 \tmp + lsx_init_upper $vr14 \tmp + lsx_init_upper $vr15 \tmp + lsx_init_upper $vr16 \tmp + lsx_init_upper $vr17 \tmp + lsx_init_upper $vr18 \tmp + lsx_init_upper $vr19 \tmp + lsx_init_upper $vr20 \tmp + lsx_init_upper $vr21 \tmp + lsx_init_upper $vr22 \tmp + lsx_init_upper $vr23 \tmp + lsx_init_upper $vr24 \tmp + lsx_init_upper $vr25 \tmp + lsx_init_upper $vr26 \tmp + lsx_init_upper $vr27 \tmp + lsx_init_upper $vr28 \tmp + lsx_init_upper $vr29 \tmp + lsx_init_upper $vr30 \tmp + lsx_init_upper $vr31 \tmp + .endm + + .macro lasx_save_data thread tmp + parse_r __tmp, \tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + /* xvst opcode is 0xb3 */ + .word (0xb3 << 22 | ((THREAD_FPR0-THREAD_FPR0) << 10) | __tmp << 5 | 0) + .word (0xb3 << 22 | ((THREAD_FPR1-THREAD_FPR0) << 10) | __tmp << 5 | 1) + .word (0xb3 << 22 | ((THREAD_FPR2-THREAD_FPR0) << 10) | __tmp << 5 | 2) + .word (0xb3 << 22 | ((THREAD_FPR3-THREAD_FPR0) << 10) | __tmp << 5 | 3) + .word (0xb3 << 22 | ((THREAD_FPR4-THREAD_FPR0) << 10) | __tmp << 5 | 4) + .word (0xb3 << 22 | ((THREAD_FPR5-THREAD_FPR0) << 10) | __tmp << 5 | 5) + .word (0xb3 << 22 | ((THREAD_FPR6-THREAD_FPR0) << 10) | __tmp << 5 | 6) + .word (0xb3 << 22 | ((THREAD_FPR7-THREAD_FPR0) << 10) | __tmp << 5 | 7) + .word (0xb3 << 22 | ((THREAD_FPR8-THREAD_FPR0) << 10) | __tmp << 5 | 8) + .word (0xb3 << 22 | ((THREAD_FPR9-THREAD_FPR0) << 10) | __tmp << 5 | 9) + .word (0xb3 << 22 | ((THREAD_FPR10-THREAD_FPR0) << 10) | __tmp << 5 | 10) + .word (0xb3 << 22 | ((THREAD_FPR11-THREAD_FPR0) << 10) | __tmp << 5 | 11) + .word (0xb3 << 22 | ((THREAD_FPR12-THREAD_FPR0) << 10) | __tmp << 5 | 12) + .word (0xb3 << 22 | ((THREAD_FPR13-THREAD_FPR0) << 10) | __tmp << 5 | 13) + .word (0xb3 << 22 | ((THREAD_FPR14-THREAD_FPR0) << 10) | __tmp << 5 | 14) + .word (0xb3 << 22 | ((THREAD_FPR15-THREAD_FPR0) << 10) | __tmp << 5 | 15) + .word (0xb3 << 22 | ((THREAD_FPR16-THREAD_FPR0) << 10) | __tmp << 5 | 16) + .word (0xb3 << 22 | ((THREAD_FPR17-THREAD_FPR0) << 10) | __tmp << 5 | 17) + .word (0xb3 << 22 | ((THREAD_FPR18-THREAD_FPR0) << 10) | __tmp << 5 | 18) + .word (0xb3 << 22 | ((THREAD_FPR19-THREAD_FPR0) << 10) | __tmp << 5 | 19) + .word (0xb3 << 22 | ((THREAD_FPR20-THREAD_FPR0) << 10) | __tmp << 5 | 20) + .word (0xb3 << 22 | ((THREAD_FPR21-THREAD_FPR0) << 10) | __tmp << 5 | 21) + .word (0xb3 << 22 | ((THREAD_FPR22-THREAD_FPR0) << 10) | __tmp << 5 | 22) + .word (0xb3 << 22 | ((THREAD_FPR23-THREAD_FPR0) << 10) | __tmp << 5 | 23) + .word (0xb3 << 22 | ((THREAD_FPR24-THREAD_FPR0) << 10) | __tmp << 5 | 24) + .word (0xb3 << 22 | ((THREAD_FPR25-THREAD_FPR0) << 10) | __tmp << 5 | 25) + .word (0xb3 << 22 | ((THREAD_FPR26-THREAD_FPR0) << 10) | __tmp << 5 | 26) + .word (0xb3 << 22 | ((THREAD_FPR27-THREAD_FPR0) << 10) | __tmp << 5 | 27) + .word (0xb3 << 22 | ((THREAD_FPR28-THREAD_FPR0) << 10) | __tmp << 5 | 28) + .word (0xb3 << 22 | ((THREAD_FPR29-THREAD_FPR0) << 10) | __tmp << 5 | 29) + .word (0xb3 << 22 | ((THREAD_FPR30-THREAD_FPR0) << 10) | __tmp << 5 | 30) + .word (0xb3 << 22 | ((THREAD_FPR31-THREAD_FPR0) << 10) | __tmp << 5 | 31) + .endm + + .macro lasx_restore_data thread tmp + parse_r __tmp, \tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \tmp, \thread, \tmp + /* xvld opcode is 0xb2 */ + .word (0xb2 << 22 | ((THREAD_FPR0-THREAD_FPR0) << 10) | __tmp << 5 | 0) + .word (0xb2 << 22 | ((THREAD_FPR1-THREAD_FPR0) << 10) | __tmp << 5 | 1) + .word (0xb2 << 22 | ((THREAD_FPR2-THREAD_FPR0) << 10) | __tmp << 5 | 2) + .word (0xb2 << 22 | ((THREAD_FPR3-THREAD_FPR0) << 10) | __tmp << 5 | 3) + .word (0xb2 << 22 | ((THREAD_FPR4-THREAD_FPR0) << 10) | __tmp << 5 | 4) + .word (0xb2 << 22 | ((THREAD_FPR5-THREAD_FPR0) << 10) | __tmp << 5 | 5) + .word (0xb2 << 22 | ((THREAD_FPR6-THREAD_FPR0) << 10) | __tmp << 5 | 6) + .word (0xb2 << 22 | ((THREAD_FPR7-THREAD_FPR0) << 10) | __tmp << 5 | 7) + .word (0xb2 << 22 | ((THREAD_FPR8-THREAD_FPR0) << 10) | __tmp << 5 | 8) + .word (0xb2 << 22 | ((THREAD_FPR9-THREAD_FPR0) << 10) | __tmp << 5 | 9) + .word (0xb2 << 22 | ((THREAD_FPR10-THREAD_FPR0) << 10) | __tmp << 5 | 10) + .word (0xb2 << 22 | ((THREAD_FPR11-THREAD_FPR0) << 10) | __tmp << 5 | 11) + .word (0xb2 << 22 | ((THREAD_FPR12-THREAD_FPR0) << 10) | __tmp << 5 | 12) + .word (0xb2 << 22 | ((THREAD_FPR13-THREAD_FPR0) << 10) | __tmp << 5 | 13) + .word (0xb2 << 22 | ((THREAD_FPR14-THREAD_FPR0) << 10) | __tmp << 5 | 14) + .word (0xb2 << 22 | ((THREAD_FPR15-THREAD_FPR0) << 10) | __tmp << 5 | 15) + .word (0xb2 << 22 | ((THREAD_FPR16-THREAD_FPR0) << 10) | __tmp << 5 | 16) + .word (0xb2 << 22 | ((THREAD_FPR17-THREAD_FPR0) << 10) | __tmp << 5 | 17) + .word (0xb2 << 22 | ((THREAD_FPR18-THREAD_FPR0) << 10) | __tmp << 5 | 18) + .word (0xb2 << 22 | ((THREAD_FPR19-THREAD_FPR0) << 10) | __tmp << 5 | 19) + .word (0xb2 << 22 | ((THREAD_FPR20-THREAD_FPR0) << 10) | __tmp << 5 | 20) + .word (0xb2 << 22 | ((THREAD_FPR21-THREAD_FPR0) << 10) | __tmp << 5 | 21) + .word (0xb2 << 22 | ((THREAD_FPR22-THREAD_FPR0) << 10) | __tmp << 5 | 22) + .word (0xb2 << 22 | ((THREAD_FPR23-THREAD_FPR0) << 10) | __tmp << 5 | 23) + .word (0xb2 << 22 | ((THREAD_FPR24-THREAD_FPR0) << 10) | __tmp << 5 | 24) + .word (0xb2 << 22 | ((THREAD_FPR25-THREAD_FPR0) << 10) | __tmp << 5 | 25) + .word (0xb2 << 22 | ((THREAD_FPR26-THREAD_FPR0) << 10) | __tmp << 5 | 26) + .word (0xb2 << 22 | ((THREAD_FPR27-THREAD_FPR0) << 10) | __tmp << 5 | 27) + .word (0xb2 << 22 | ((THREAD_FPR28-THREAD_FPR0) << 10) | __tmp << 5 | 28) + .word (0xb2 << 22 | ((THREAD_FPR29-THREAD_FPR0) << 10) | __tmp << 5 | 29) + .word (0xb2 << 22 | ((THREAD_FPR30-THREAD_FPR0) << 10) | __tmp << 5 | 30) + .word (0xb2 << 22 | ((THREAD_FPR31-THREAD_FPR0) << 10) | __tmp << 5 | 31) + .endm + + .macro lasx_save_all thread tmp0 tmp1 + fpu_save_cc \thread, \tmp0, \tmp1 + fpu_save_csr \thread, \tmp0 + lasx_save_data \thread, \tmp0 + .endm + + .macro lasx_restore_all thread tmp0 tmp1 + lasx_restore_data \thread, \tmp0 + fpu_restore_cc \thread, \tmp0, \tmp1 + fpu_restore_csr \thread, \tmp0 + .endm + + .macro lasx_save_upper xd base tmp off + /* Nothing */ + .endm + + .macro lasx_save_all_upper thread base tmp + /* Nothing */ + .endm + + .macro lasx_restore_upper xd base tmp off + parse_xr __xd, \xd + parse_xr __xt, \tmp + parse_r __base, \base + /* vld opcode is 0xb0 */ + .word (0xb0 << 22 | (\off+16) << 10 | __base << 5 | __xt) + /* xvpermi.q opcode is 0x1dfb */ + .word (0x1dfb << 18 | 0x2 << 10 | __xt << 5 | __xd) + .endm + + .macro lasx_restore_all_upper thread base tmp + li.w \tmp, THREAD_FPR0 + PTR_ADD \base, \thread, \tmp + /* Save $vr31, xvpickve2gr opcode is 0x76efe */ + .word (0x76efe << 12 | 0 << 10 | 31 << 5 | 0x11) + .word (0x76efe << 12 | 1 << 10 | 31 << 5 | 0x12) + lasx_restore_upper $xr0, \base, $xr31, (THREAD_FPR0-THREAD_FPR0) + lasx_restore_upper $xr1, \base, $xr31, (THREAD_FPR1-THREAD_FPR0) + lasx_restore_upper $xr2, \base, $xr31, (THREAD_FPR2-THREAD_FPR0) + lasx_restore_upper $xr3, \base, $xr31, (THREAD_FPR3-THREAD_FPR0) + lasx_restore_upper $xr4, \base, $xr31, (THREAD_FPR4-THREAD_FPR0) + lasx_restore_upper $xr5, \base, $xr31, (THREAD_FPR5-THREAD_FPR0) + lasx_restore_upper $xr6, \base, $xr31, (THREAD_FPR6-THREAD_FPR0) + lasx_restore_upper $xr7, \base, $xr31, (THREAD_FPR7-THREAD_FPR0) + lasx_restore_upper $xr8, \base, $xr31, (THREAD_FPR8-THREAD_FPR0) + lasx_restore_upper $xr9, \base, $xr31, (THREAD_FPR9-THREAD_FPR0) + lasx_restore_upper $xr10, \base, $xr31, (THREAD_FPR10-THREAD_FPR0) + lasx_restore_upper $xr11, \base, $xr31, (THREAD_FPR11-THREAD_FPR0) + lasx_restore_upper $xr12, \base, $xr31, (THREAD_FPR12-THREAD_FPR0) + lasx_restore_upper $xr13, \base, $xr31, (THREAD_FPR13-THREAD_FPR0) + lasx_restore_upper $xr14, \base, $xr31, (THREAD_FPR14-THREAD_FPR0) + lasx_restore_upper $xr15, \base, $xr31, (THREAD_FPR15-THREAD_FPR0) + lasx_restore_upper $xr16, \base, $xr31, (THREAD_FPR16-THREAD_FPR0) + lasx_restore_upper $xr17, \base, $xr31, (THREAD_FPR17-THREAD_FPR0) + lasx_restore_upper $xr18, \base, $xr31, (THREAD_FPR18-THREAD_FPR0) + lasx_restore_upper $xr19, \base, $xr31, (THREAD_FPR19-THREAD_FPR0) + lasx_restore_upper $xr20, \base, $xr31, (THREAD_FPR20-THREAD_FPR0) + lasx_restore_upper $xr21, \base, $xr31, (THREAD_FPR21-THREAD_FPR0) + lasx_restore_upper $xr22, \base, $xr31, (THREAD_FPR22-THREAD_FPR0) + lasx_restore_upper $xr23, \base, $xr31, (THREAD_FPR23-THREAD_FPR0) + lasx_restore_upper $xr24, \base, $xr31, (THREAD_FPR24-THREAD_FPR0) + lasx_restore_upper $xr25, \base, $xr31, (THREAD_FPR25-THREAD_FPR0) + lasx_restore_upper $xr26, \base, $xr31, (THREAD_FPR26-THREAD_FPR0) + lasx_restore_upper $xr27, \base, $xr31, (THREAD_FPR27-THREAD_FPR0) + lasx_restore_upper $xr28, \base, $xr31, (THREAD_FPR28-THREAD_FPR0) + lasx_restore_upper $xr29, \base, $xr31, (THREAD_FPR29-THREAD_FPR0) + lasx_restore_upper $xr30, \base, $xr31, (THREAD_FPR30-THREAD_FPR0) + lasx_restore_upper $xr31, \base, $xr31, (THREAD_FPR31-THREAD_FPR0) + /* Restore $vr31, xvinsgr2vr opcode is 0x76ebe */ + .word (0x76ebe << 12 | 0 << 10 | 0x11 << 5 | 31) + .word (0x76ebe << 12 | 1 << 10 | 0x12 << 5 | 31) + .endm + + .macro lasx_init_upper xd tmp + parse_xr __xd, \xd + parse_r __tmp, \tmp + /* xvinsgr2vr opcode is 0x76ebe */ + .word (0x76ebe << 12 | 2 << 10 | __tmp << 5 | __xd) + .word (0x76ebe << 12 | 3 << 10 | __tmp << 5 | __xd) + .endm + + .macro lasx_init_all_upper tmp + not \tmp, zero + lasx_init_upper $xr0 \tmp + lasx_init_upper $xr1 \tmp + lasx_init_upper $xr2 \tmp + lasx_init_upper $xr3 \tmp + lasx_init_upper $xr4 \tmp + lasx_init_upper $xr5 \tmp + lasx_init_upper $xr6 \tmp + lasx_init_upper $xr7 \tmp + lasx_init_upper $xr8 \tmp + lasx_init_upper $xr9 \tmp + lasx_init_upper $xr10 \tmp + lasx_init_upper $xr11 \tmp + lasx_init_upper $xr12 \tmp + lasx_init_upper $xr13 \tmp + lasx_init_upper $xr14 \tmp + lasx_init_upper $xr15 \tmp + lasx_init_upper $xr16 \tmp + lasx_init_upper $xr17 \tmp + lasx_init_upper $xr18 \tmp + lasx_init_upper $xr19 \tmp + lasx_init_upper $xr20 \tmp + lasx_init_upper $xr21 \tmp + lasx_init_upper $xr22 \tmp + lasx_init_upper $xr23 \tmp + lasx_init_upper $xr24 \tmp + lasx_init_upper $xr25 \tmp + lasx_init_upper $xr26 \tmp + lasx_init_upper $xr27 \tmp + lasx_init_upper $xr28 \tmp + lasx_init_upper $xr29 \tmp + lasx_init_upper $xr30 \tmp + lasx_init_upper $xr31 \tmp + .endm + +.macro jr dst + jirl zero, \dst, 0 +.endm + +.macro jalr dst + jirl ra, \dst, 0 +.endm + +.macro not dst src + nor \dst, \src, zero +.endm + +.macro bgt r0 r1 label + blt \r1, \r0, \label +.endm + +.macro bltz r0 label + blt \r0, zero, \label +.endm + +.macro bgez r0 label + bge \r0, zero, \label +.endm + +#endif /* _ASM_ASMMACRO_H */ diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h new file mode 100644 index 0000000000000000000000000000000000000000..f054952b122b69d185f4810b75ae6e83f948e043 --- /dev/null +++ b/arch/loongarch/include/asm/atomic.h @@ -0,0 +1,415 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Atomic operations that C can't guarantee us. Useful for + * resource counting etc.. + * + * But use these as seldom as possible since they are much more slower + * than regular operations. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_ATOMIC_H +#define _ASM_ATOMIC_H + +#include +#include +#include +#include + +#if __SIZEOF_LONG__ == 4 +#define __LL "ll.w " +#define __SC "sc.w " +#define __AMADD "amadd.w " +#define __AMAND_DB "amand_db.w " +#define __AMOR_DB "amor_db.w " +#define __AMXOR_DB "amxor_db.w " +#elif __SIZEOF_LONG__ == 8 +#define __LL "ll.d " +#define __SC "sc.d " +#define __AMADD "amadd.d " +#define __AMAND_DB "amand_db.d " +#define __AMOR_DB "amor_db.d " +#define __AMXOR_DB "amxor_db.d " +#endif + +#define ATOMIC_INIT(i) { (i) } + +/* + * atomic_read - read atomic variable + * @v: pointer of type atomic_t + * + * Atomically reads the value of @v. + */ +#define atomic_read(v) READ_ONCE((v)->counter) + +/* + * atomic_set - set atomic variable + * @v: pointer of type atomic_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +#define atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) + +#define ATOMIC_OP(op, I, asm_op) \ +static __inline__ void atomic_##op(int i, atomic_t * v) \ +{ \ + __asm__ __volatile__( \ + "am"#asm_op".w" " $zero, %1, %0 \n" \ + : "+ZB" (v->counter) \ + : "r" (I) \ + : "memory"); \ +} + +#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ +static __inline__ int atomic_##op##_return##suffix(int i, atomic_t * v) \ +{ \ + int result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".w" " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result c_op I; \ +} + +#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \ +static __inline__ int atomic_fetch_##op##suffix(int i, atomic_t * v) \ +{ \ + int result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".w" " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op, I, asm_op, c_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \ + ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ + ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC_OPS(add, i, add, +) +ATOMIC_OPS(sub, -i, add, +) + +#define atomic_add_return atomic_add_return +#define atomic_add_return_acquire atomic_add_return +#define atomic_add_return_release atomic_add_return +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return atomic_sub_return +#define atomic_sub_return_acquire atomic_sub_return +#define atomic_sub_return_release atomic_sub_return +#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define atomic_fetch_add atomic_fetch_add +#define atomic_fetch_add_acquire atomic_fetch_add +#define atomic_fetch_add_release atomic_fetch_add +#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed +#define atomic_fetch_sub atomic_fetch_sub +#define atomic_fetch_sub_acquire atomic_fetch_sub +#define atomic_fetch_sub_release atomic_fetch_sub +#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed + +#undef ATOMIC_OPS + +#define ATOMIC_OPS(op, I, asm_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC_OPS(and, i, and) +ATOMIC_OPS(or, i, or) +ATOMIC_OPS(xor, i, xor) + +#define atomic_fetch_and atomic_fetch_and +#define atomic_fetch_and_acquire atomic_fetch_and +#define atomic_fetch_and_release atomic_fetch_and +#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed +#define atomic_fetch_or atomic_fetch_or +#define atomic_fetch_or_acquire atomic_fetch_or +#define atomic_fetch_or_release atomic_fetch_or +#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed +#define atomic_fetch_xor atomic_fetch_xor +#define atomic_fetch_xor_acquire atomic_fetch_xor +#define atomic_fetch_xor_release atomic_fetch_xor +#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int prev, rc; + + __asm__ __volatile__ ( + "0: ll.w %[p], %[c]\n" + " beq %[p], %[u], 1f\n" + " add.w %[rc], %[p], %[a]\n" + " sc.w %[rc], %[c]\n" + " beqz %[rc], 0b\n" + " b 2f\n" + "1:\n" + __WEAK_LLSC_MB + "2:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), + [c]"=ZB" (v->counter) + : [a]"r" (a), [u]"r" (u) + : "memory"); + + return prev; +} +#define atomic_fetch_add_unless atomic_fetch_add_unless + +/* + * atomic_sub_if_positive - conditionally subtract integer from atomic variable + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically test @v and subtract @i if @v is greater or equal than @i. + * The function returns the old value of @v minus @i. + */ +static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) +{ + int result; + int temp; + + if (__builtin_constant_p(i)) { + __asm__ __volatile__( + "1: ll.w %1, %2 # atomic_sub_if_positive\n" + " addi.w %0, %1, %3 \n" + " or %1, %0, $zero \n" + " blt %0, $zero, 2f \n" + " sc.w %1, %2 \n" + " beq $zero, %1, 1b \n" + "2: \n" + __WEAK_LLSC_MB + : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) + : "I" (-i)); + } else { + __asm__ __volatile__( + "1: ll.w %1, %2 # atomic_sub_if_positive\n" + " sub.w %0, %1, %3 \n" + " or %1, %0, $zero \n" + " blt %0, $zero, 2f \n" + " sc.w %1, %2 \n" + " beq $zero, %1, 1b \n" + "2: \n" + __WEAK_LLSC_MB + : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) + : "r" (i)); + } + + return result; +} + +#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) + +/* + * atomic_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic_t + */ +#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) + +#ifdef CONFIG_64BIT + +#define ATOMIC64_INIT(i) { (i) } + +/* + * atomic64_read - read atomic variable + * @v: pointer of type atomic64_t + * + */ +#define atomic64_read(v) READ_ONCE((v)->counter) + +/* + * atomic64_set - set atomic variable + * @v: pointer of type atomic64_t + * @i: required value + */ +#define atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) + +#define ATOMIC64_OP(op, I, asm_op) \ +static __inline__ void atomic64_##op(long i, atomic64_t * v) \ +{ \ + __asm__ __volatile__( \ + "am"#asm_op".d " " $zero, %1, %0 \n" \ + : "+ZB" (v->counter) \ + : "r" (I) \ + : "memory"); \ +} + +#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ +static __inline__ long atomic64_##op##_return##suffix(long i, atomic64_t * v) \ +{ \ + long result; \ + __asm__ __volatile__( \ + "am"#asm_op#mb".d " " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result c_op I; \ +} + +#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \ +static __inline__ long atomic64_fetch_##op##suffix(long i, atomic64_t * v) \ +{ \ + long result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".d " " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result; \ +} + +#define ATOMIC64_OPS(op, I, asm_op, c_op) \ + ATOMIC64_OP(op, I, asm_op) \ + ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \ + ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ + ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC64_OPS(add, i, add, +) +ATOMIC64_OPS(sub, -i, add, +) + +#define atomic64_add_return atomic64_add_return +#define atomic64_add_return_acquire atomic64_add_return +#define atomic64_add_return_release atomic64_add_return +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return atomic64_sub_return +#define atomic64_sub_return_acquire atomic64_sub_return +#define atomic64_sub_return_release atomic64_sub_return +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define atomic64_fetch_add atomic64_fetch_add +#define atomic64_fetch_add_acquire atomic64_fetch_add +#define atomic64_fetch_add_release atomic64_fetch_add +#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed +#define atomic64_fetch_sub atomic64_fetch_sub +#define atomic64_fetch_sub_acquire atomic64_fetch_sub +#define atomic64_fetch_sub_release atomic64_fetch_sub +#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed + +#undef ATOMIC64_OPS + +#define ATOMIC64_OPS(op, I, asm_op) \ + ATOMIC64_OP(op, I, asm_op) \ + ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC64_OPS(and, i, and) +ATOMIC64_OPS(or, i, or) +ATOMIC64_OPS(xor, i, xor) + +#define atomic64_fetch_and atomic64_fetch_and +#define atomic64_fetch_and_acquire atomic64_fetch_and +#define atomic64_fetch_and_release atomic64_fetch_and +#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed +#define atomic64_fetch_or atomic64_fetch_or +#define atomic64_fetch_or_acquire atomic64_fetch_or +#define atomic64_fetch_or_release atomic64_fetch_or +#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed +#define atomic64_fetch_xor atomic64_fetch_xor +#define atomic64_fetch_xor_acquire atomic64_fetch_xor +#define atomic64_fetch_xor_release atomic64_fetch_xor +#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP + +static __inline__ long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) +{ + long prev, rc; + + __asm__ __volatile__ ( + "0: ll.d %[p], %[c]\n" + " beq %[p], %[u], 1f\n" + " add.d %[rc], %[p], %[a]\n" + " sc.d %[rc], %[c]\n" + " beqz %[rc], 0b\n" + " b 2f\n" + "1:\n" + __WEAK_LLSC_MB + "2:\n" + : [p]"=&r" (prev), [rc]"=&r" (rc), + [c] "=ZB" (v->counter) + : [a]"r" (a), [u]"r" (u) + : "memory"); + + return prev; +} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless + +/* + * atomic64_sub_if_positive - conditionally subtract integer from atomic + * variable + * @i: integer value to subtract + * @v: pointer of type atomic64_t + * + * Atomically test @v and subtract @i if @v is greater or equal than @i. + * The function returns the old value of @v minus @i. + */ +static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) +{ + long result; + long temp; + + if (__builtin_constant_p(i)) { + __asm__ __volatile__( + "1: ll.d %1, %2 # atomic64_sub_if_positive \n" + " addi.d %0, %1, %3 \n" + " or %1, %0, $zero \n" + " blt %0, $zero, 2f \n" + " sc.d %1, %2 \n" + " beq %1, $zero, 1b \n" + "2: \n" + __WEAK_LLSC_MB + : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) + : "I" (-i)); + } else { + __asm__ __volatile__( + "1: ll.d %1, %2 # atomic64_sub_if_positive \n" + " sub.d %0, %1, %3 \n" + " or %1, %0, $zero \n" + " blt %0, $zero, 2f \n" + " sc.d %1, %2 \n" + " beq %1, $zero, 1b \n" + "2: \n" + __WEAK_LLSC_MB + : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) + : "r" (i)); + } + + return result; +} + +#define atomic64_cmpxchg(v, o, n) \ + ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) +#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new))) + +/* + * atomic64_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic64_t + */ +#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v) + +#endif /* CONFIG_64BIT */ + +#endif /* _ASM_ATOMIC_H */ diff --git a/arch/loongarch/include/asm/barrier.h b/arch/loongarch/include/asm/barrier.h new file mode 100644 index 0000000000000000000000000000000000000000..f33162d4e40505a1823e6b9b4082c8de4b345f97 --- /dev/null +++ b/arch/loongarch/include/asm/barrier.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_BARRIER_H +#define __ASM_BARRIER_H + +#include + +/* + * Hint encoding: + * + * Bit4: ordering or completion (0: completion, 1: ordering) + * Bit3: barrier for previous read (0: true, 1: false) + * Bit2: barrier for previous write (0: true, 1: false) + * Bit1: barrier for succeeding read (0: true, 1: false) + * Bit0: barrier for succeeding write (0: true, 1: false) + * + * Hint 0x700: barrier for "read after read" from the same address + */ + +#define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory") + +#define crwrw 0b00000 +#define cr_r_ 0b00101 +#define c_w_w 0b01010 + +#define orwrw 0b10000 +#define or_r_ 0b10101 +#define o_w_w 0b11010 + +#define orw_w 0b10010 +#define or_rw 0b10100 + +#define c_sync() DBAR(crwrw) +#define c_rsync() DBAR(cr_r_) +#define c_wsync() DBAR(c_w_w) + +#define o_sync() DBAR(orwrw) +#define o_rsync() DBAR(or_r_) +#define o_wsync() DBAR(o_w_w) + +#define ldacq_mb() DBAR(or_rw) +#define strel_mb() DBAR(orw_w) + +#define mb() c_sync() +#define rmb() c_rsync() +#define wmb() c_wsync() +#define iob() c_sync() +#define wbflush() c_sync() + +#define __smp_mb() o_sync() +#define __smp_rmb() o_rsync() +#define __smp_wmb() o_wsync() + +#ifdef CONFIG_SMP +#define __WEAK_LLSC_MB " dbar 0x700 \n" +#else +#define __WEAK_LLSC_MB " \n" +#endif + +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +/** + * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise + * @index: array element index + * @size: number of elements in array + * + * Returns: + * 0 - (@index < @size) + */ +#define array_index_mask_nospec array_index_mask_nospec +static inline unsigned long array_index_mask_nospec(unsigned long index, + unsigned long size) +{ + unsigned long mask; + + __asm__ __volatile__( + "sltu %0, %1, %2\n\t" +#if (__SIZEOF_LONG__ == 4) + "sub.w %0, $r0, %0\n\t" +#elif (__SIZEOF_LONG__ == 8) + "sub.d %0, $r0, %0\n\t" +#endif + : "=r" (mask) + : "r" (index), "r" (size) + :); + + return mask; +} + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + ldacq_mb(); \ + ___p1; \ +}) + +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + strel_mb(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_store_mb(p, v) \ +do { \ + union { typeof(p) __val; char __c[1]; } __u = \ + { .__val = (__force typeof(p)) (v) }; \ + unsigned long __tmp; \ + switch (sizeof(p)) { \ + case 1: \ + *(volatile __u8 *)&p = *(__u8 *)__u.__c; \ + __smp_mb(); \ + break; \ + case 2: \ + *(volatile __u16 *)&p = *(__u16 *)__u.__c; \ + __smp_mb(); \ + break; \ + case 4: \ + __asm__ __volatile__( \ + "amswap_db.w %[tmp], %[val], %[mem] \n" \ + : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \ + : [val] "r" (*(__u32 *)__u.__c) \ + : ); \ + break; \ + case 8: \ + __asm__ __volatile__( \ + "amswap_db.d %[tmp], %[val], %[mem] \n" \ + : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \ + : [val] "r" (*(__u64 *)__u.__c) \ + : ); \ + break; \ + } \ +} while (0) + +#include + +#endif /* __ASM_BARRIER_H */ diff --git a/arch/loongarch/include/asm/bitops.h b/arch/loongarch/include/asm/bitops.h new file mode 100644 index 0000000000000000000000000000000000000000..c7384870267bf948a46e2bf9fcdf038513229712 --- /dev/null +++ b/arch/loongarch/include/asm/bitops.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_BITOPS_H +#define _ASM_BITOPS_H + +#include + +#ifndef _LINUX_BITOPS_H +#error only can be included directly +#endif + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#endif /* _ASM_BITOPS_H */ diff --git a/arch/loongarch/include/asm/bitrev.h b/arch/loongarch/include/asm/bitrev.h new file mode 100644 index 0000000000000000000000000000000000000000..38147173f2adffb5ca35eba25cc04bc6c2909e2e --- /dev/null +++ b/arch/loongarch/include/asm/bitrev.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __LOONGARCH_ASM_BITREV_H__ +#define __LOONGARCH_ASM_BITREV_H__ + +#include + +static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x) +{ + u32 ret; + + asm("bitrev.4b %0, %1" : "=r"(ret) : "r"(__swab32(x))); + return ret; +} + +static __always_inline __attribute_const__ u16 __arch_bitrev16(u16 x) +{ + u16 ret; + + asm("bitrev.4b %0, %1" : "=r"(ret) : "r"(__swab16(x))); + return ret; +} + +static __always_inline __attribute_const__ u8 __arch_bitrev8(u8 x) +{ + u8 ret; + + asm("bitrev.4b %0, %1" : "=r"(ret) : "r"(x)); + return ret; +} + +#endif /* __LOONGARCH_ASM_BITREV_H__ */ diff --git a/arch/loongarch/include/asm/bootinfo.h b/arch/loongarch/include/asm/bootinfo.h new file mode 100644 index 0000000000000000000000000000000000000000..14b4b425a513289ec7227d2276f91aa2335e4ef7 --- /dev/null +++ b/arch/loongarch/include/asm/bootinfo.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_BOOTINFO_H +#define _ASM_BOOTINFO_H + +#include +#include + +const char *get_system_type(void); + +extern void memblock_init(void); +extern void detect_memory_region(phys_addr_t start, phys_addr_t sz_min, phys_addr_t sz_max); + +extern void early_init(void); +extern void init_environ(void); +extern void platform_init(void); +extern void plat_mem_setup(void); +extern int __init init_numa_memory(void); + +struct loongson_board_info { + int bios_size; + const char *bios_vendor; + const char *bios_version; + const char *bios_release_date; + const char *board_name; + const char *board_vendor; +}; + +struct loongson_system_configuration { + int nr_cpus; + int nr_nodes; + int nr_io_pics; + int boot_cpu_id; + int cores_per_node; + int cores_per_package; + const char *cpuname; + u64 suspend_addr; +}; + +extern u64 efi_system_table; +extern struct loongson_board_info b_info; +extern struct loongson_system_configuration loongson_sysconf; +extern unsigned long fw_arg0, fw_arg1, fw_arg2; + +#endif /* _ASM_BOOTINFO_H */ diff --git a/arch/loongarch/include/asm/branch.h b/arch/loongarch/include/asm/branch.h new file mode 100644 index 0000000000000000000000000000000000000000..830279859f4c28ec32a0430d24181d4148b53553 --- /dev/null +++ b/arch/loongarch/include/asm/branch.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_BRANCH_H +#define _ASM_BRANCH_H + +#include + +static inline unsigned long exception_era(struct pt_regs *regs) +{ + return regs->csr_era; +} + +static inline int compute_return_era(struct pt_regs *regs) +{ + regs->csr_era += 4; + return 0; +} + +#endif /* _ASM_BRANCH_H */ diff --git a/arch/loongarch/include/asm/bug.h b/arch/loongarch/include/asm/bug.h new file mode 100644 index 0000000000000000000000000000000000000000..b200d4e261383fdeae8bbb187e95487ae1bb8857 --- /dev/null +++ b/arch/loongarch/include/asm/bug.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_BUG_H +#define __ASM_BUG_H + +#include +#include + +#ifndef CONFIG_DEBUG_BUGVERBOSE +#define _BUGVERBOSE_LOCATION(file, line) +#else +#define __BUGVERBOSE_LOCATION(file, line) \ + .pushsection .rodata.str, "aMS", @progbits, 1; \ + 10002: .string file; \ + .popsection; \ + \ + .long 10002b - 10000b; \ + .short line; +#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) +#endif + +#ifndef CONFIG_GENERIC_BUG +#define __BUG_ENTRY(flags) +#else +#define __BUG_ENTRY(flags) \ + .pushsection __bug_table, "aw"; \ + .align 2; \ + 10000: .long 10001f - 10000b; \ + _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ + .short flags; \ + .popsection; \ + 10001: +#endif + +#define ASM_BUG_FLAGS(flags) \ + __BUG_ENTRY(flags) \ + break BRK_BUG + +#define ASM_BUG() ASM_BUG_FLAGS(0) + +#define __BUG_FLAGS(flags) \ + asm_inline volatile (__stringify(ASM_BUG_FLAGS(flags))); + +#define __WARN_FLAGS(flags) \ +do { \ + instrumentation_begin(); \ + __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \ + annotate_reachable(); \ + instrumentation_end(); \ +} while (0) + +#define BUG() \ +do { \ + instrumentation_begin(); \ + __BUG_FLAGS(0); \ + unreachable(); \ +} while (0) + +#define HAVE_ARCH_BUG + +#include + +#endif /* __ASM_BUG_H */ diff --git a/arch/loongarch/include/asm/cache.h b/arch/loongarch/include/asm/cache.h new file mode 100644 index 0000000000000000000000000000000000000000..62eb6271af428e0b2c159aba3a05f367215ea632 --- /dev/null +++ b/arch/loongarch/include/asm/cache.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_CACHE_H +#define _ASM_CACHE_H + +#define L1_CACHE_SHIFT CONFIG_L1_CACHE_SHIFT +#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) + +#define __read_mostly __attribute__((__section__(".data..read_mostly"))) + +#endif /* _ASM_CACHE_H */ diff --git a/arch/loongarch/include/asm/cacheflush.h b/arch/loongarch/include/asm/cacheflush.h new file mode 100644 index 0000000000000000000000000000000000000000..27f9078b6a5ba96343a7d84faee773215a8c6c8f --- /dev/null +++ b/arch/loongarch/include/asm/cacheflush.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_CACHEFLUSH_H +#define _ASM_CACHEFLUSH_H + +#include +#include +#include + +static inline bool cache_present(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_PRESENT; +} + +static inline bool cache_private(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_PRIVATE; +} + +static inline bool cache_inclusive(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_INCLUSIVE; +} + +static inline unsigned int cpu_last_level_cache_line_size(void) +{ + int cache_present = boot_cpu_data.cache_leaves_present; + + return boot_cpu_data.cache_leaves[cache_present - 1].linesz; +} + +asmlinkage void __flush_cache_all(void); +void local_flush_icache_range(unsigned long start, unsigned long end); + +#define flush_icache_range local_flush_icache_range +#define flush_icache_user_range local_flush_icache_range + +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 + +#define flush_cache_all() do { } while (0) +#define flush_cache_mm(mm) do { } while (0) +#define flush_cache_dup_mm(mm) do { } while (0) +#define flush_cache_range(vma, start, end) do { } while (0) +#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) +#define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vunmap(start, end) do { } while (0) +#define flush_icache_page(vma, page) do { } while (0) +#define flush_icache_user_page(vma, page, addr, len) do { } while (0) +#define flush_dcache_page(page) do { } while (0) +#define flush_dcache_mmap_lock(mapping) do { } while (0) +#define flush_dcache_mmap_unlock(mapping) do { } while (0) + +#define cache_op(op,addr) \ + __asm__ __volatile__( \ + " cacop %0, %1 \n" \ + : \ + : "i" (op), "ZC" (*(unsigned char *)(addr))) + +static inline void flush_cache_line(int leaf, unsigned long addr) +{ + switch (leaf) { + case Cache_LEAF0: + cache_op(Index_Writeback_Inv_LEAF0, addr); + break; + case Cache_LEAF1: + cache_op(Index_Writeback_Inv_LEAF1, addr); + break; + case Cache_LEAF2: + cache_op(Index_Writeback_Inv_LEAF2, addr); + break; + case Cache_LEAF3: + cache_op(Index_Writeback_Inv_LEAF3, addr); + break; + case Cache_LEAF4: + cache_op(Index_Writeback_Inv_LEAF4, addr); + break; + case Cache_LEAF5: + cache_op(Index_Writeback_Inv_LEAF5, addr); + break; + default: + break; + } +} + +#include + +#endif /* _ASM_CACHEFLUSH_H */ diff --git a/arch/loongarch/include/asm/cacheops.h b/arch/loongarch/include/asm/cacheops.h new file mode 100644 index 0000000000000000000000000000000000000000..51ff636b8f8e1a1077abbb5961b0785cd8b9a504 --- /dev/null +++ b/arch/loongarch/include/asm/cacheops.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * Cache operations for the cache instruction. + */ +#ifndef __ASM_CACHEOPS_H +#define __ASM_CACHEOPS_H + +/* + * Most cache ops are split into a 3 bit field identifying the cache, and a 2 + * bit field identifying the cache operation. + */ +#define CacheOp_Cache 0x07 +#define CacheOp_Op 0x18 + +#define Cache_LEAF0 0x00 +#define Cache_LEAF1 0x01 +#define Cache_LEAF2 0x02 +#define Cache_LEAF3 0x03 +#define Cache_LEAF4 0x04 +#define Cache_LEAF5 0x05 + +#define Index_Invalidate 0x08 +#define Index_Writeback_Inv 0x08 +#define Hit_Invalidate 0x10 +#define Hit_Writeback_Inv 0x10 +#define CacheOp_User_Defined 0x18 + +#define Index_Writeback_Inv_LEAF0 (Cache_LEAF0 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF1 (Cache_LEAF1 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF2 (Cache_LEAF2 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF3 (Cache_LEAF3 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF4 (Cache_LEAF4 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF5 (Cache_LEAF5 | Index_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF0 (Cache_LEAF0 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF1 (Cache_LEAF1 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF2 (Cache_LEAF2 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF3 (Cache_LEAF3 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF4 (Cache_LEAF4 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF5 (Cache_LEAF5 | Hit_Writeback_Inv) + +#endif /* __ASM_CACHEOPS_H */ diff --git a/arch/loongarch/include/asm/checksum.h b/arch/loongarch/include/asm/checksum.h new file mode 100644 index 0000000000000000000000000000000000000000..cabbf6af44c43acd6be89ed60322b05e6fce3007 --- /dev/null +++ b/arch/loongarch/include/asm/checksum.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2016 ARM Ltd. + * Copyright (C) 2023 Loongson Technology Corporation Limited + */ +#ifndef __ASM_CHECKSUM_H +#define __ASM_CHECKSUM_H + +#include +#include + +#define _HAVE_ARCH_IPV6_CSUM +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum); + +/* + * turns a 32-bit partial checksum (e.g. from csum_partial) into a + * 1's complement 16-bit checksum. + */ +static inline __sum16 csum_fold(__wsum sum) +{ + u32 tmp = (__force u32)sum; + + /* + * swap the two 16-bit halves of sum + * if there is a carry from adding the two 16-bit halves, + * it will carry from the lower half into the upper half, + * giving us the correct sum in the upper half. + */ + return (__force __sum16)(~(tmp + rol32(tmp, 16)) >> 16); +} +#define csum_fold csum_fold + +/* + * This is a version of ip_compute_csum() optimized for IP headers, + * which always checksum on 4 octet boundaries. ihl is the number + * of 32-bit words and is always >= 5. + */ +static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + u64 sum; + __uint128_t tmp; + int n = ihl; /* we want it signed */ + + tmp = *(const __uint128_t *)iph; + iph += 16; + n -= 4; + tmp += ((tmp >> 64) | (tmp << 64)); + sum = tmp >> 64; + do { + sum += *(const u32 *)iph; + iph += 4; + } while (--n > 0); + + sum += ror64(sum, 32); + return csum_fold((__force __wsum)(sum >> 32)); +} +#define ip_fast_csum ip_fast_csum + +extern unsigned int do_csum(const unsigned char *buff, int len); +#define do_csum do_csum + +#include + +#endif /* __ASM_CHECKSUM_H */ diff --git a/arch/loongarch/include/asm/clocksource.h b/arch/loongarch/include/asm/clocksource.h new file mode 100644 index 0000000000000000000000000000000000000000..803f0b42b28ca7fd8c0403d538416724d0e59984 --- /dev/null +++ b/arch/loongarch/include/asm/clocksource.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#ifndef __ASM_CLOCKSOURCE_H +#define __ASM_CLOCKSOURCE_H + +#include + +#endif /* __ASM_CLOCKSOURCE_H */ diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h new file mode 100644 index 0000000000000000000000000000000000000000..ee79e10d8b216f6fe412ca7f93a1b159bb57d147 --- /dev/null +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_CMPXCHG_H +#define __ASM_CMPXCHG_H + +#include +#include + +#define __xchg_asm(amswap_db, m, val) \ +({ \ + __typeof(val) __ret; \ + \ + __asm__ __volatile__ ( \ + " "amswap_db" %1, %z2, %0 \n" \ + : "+ZB" (*m), "=&r" (__ret) \ + : "Jr" (val) \ + : "memory"); \ + \ + __ret; \ +}) + +extern unsigned long __xchg_small(volatile void *ptr, unsigned long x, + unsigned int size); + +static __always_inline unsigned long +__xchg(volatile void *ptr, unsigned long x, int size) +{ + switch (size) { + case 1: + case 2: + return __xchg_small(ptr, x, size); + + case 4: + return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); + + case 8: + return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); + + default: + BUILD_BUG(); + } + + return 0; +} + +#define xchg(ptr, x) \ +({ \ + __typeof__(*(ptr)) __res; \ + \ + __res = (__typeof__(*(ptr))) \ + __xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \ + \ + __res; \ +}) + +#define __cmpxchg_asm(ld, st, m, old, new) \ +({ \ + __typeof(old) __ret; \ + \ + __asm__ __volatile__( \ + "1: " ld " %0, %2 # __cmpxchg_asm \n" \ + " bne %0, %z3, 2f \n" \ + " or $t0, %z4, $zero \n" \ + " " st " $t0, %1 \n" \ + " beq $zero, $t0, 1b \n" \ + "2: \n" \ + __WEAK_LLSC_MB \ + : "=&r" (__ret), "=ZB"(*m) \ + : "ZB"(*m), "Jr" (old), "Jr" (new) \ + : "t0", "memory"); \ + \ + __ret; \ +}) + +extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, + unsigned long new, unsigned int size); + +static __always_inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) +{ + switch (size) { + case 1: + case 2: + return __cmpxchg_small(ptr, old, new, size); + + case 4: + return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr, + (u32)old, new); + + case 8: + return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr, + (u64)old, new); + + default: + BUILD_BUG(); + } + + return 0; +} + +#define cmpxchg_local(ptr, old, new) \ + ((__typeof__(*(ptr))) \ + __cmpxchg((ptr), \ + (unsigned long)(__typeof__(*(ptr)))(old), \ + (unsigned long)(__typeof__(*(ptr)))(new), \ + sizeof(*(ptr)))) + +#define cmpxchg(ptr, old, new) \ +({ \ + __typeof__(*(ptr)) __res; \ + \ + __res = cmpxchg_local((ptr), (old), (new)); \ + \ + __res; \ +}) + +#ifdef CONFIG_64BIT +#define cmpxchg64_local(ptr, o, n) \ + ({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg_local((ptr), (o), (n)); \ + }) + +#define cmpxchg64(ptr, o, n) \ + ({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg((ptr), (o), (n)); \ + }) +#else +#include +#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#endif + +#endif /* __ASM_CMPXCHG_H */ diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h new file mode 100644 index 0000000000000000000000000000000000000000..cf96d843cd97b87a06719ee953811de00ed7c2dd --- /dev/null +++ b/arch/loongarch/include/asm/cpu-features.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_CPU_FEATURES_H +#define __ASM_CPU_FEATURES_H + +#include +#include + +#define cpu_opt(opt) (cpu_data[0].options & (opt)) +#define cpu_has(feat) (cpu_data[0].options & BIT_ULL(feat)) + +#define cpu_has_loongarch (cpu_has_loongarch32 | cpu_has_loongarch64) +#define cpu_has_loongarch32 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_32BIT) +#define cpu_has_loongarch64 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) + +#ifdef CONFIG_32BIT +# define cpu_has_64bits (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) +# define cpu_vabits 31 +# define cpu_pabits 31 +#endif + +#ifdef CONFIG_64BIT +# define cpu_has_64bits 1 +# define cpu_vabits cpu_data[0].vabits +# define cpu_pabits cpu_data[0].pabits +# define __NEED_ADDRBITS_PROBE +#endif + +/* + * SMP assumption: Options of CPU 0 are a superset of all processors. + * This is true for all known LoongArch systems. + */ +#define cpu_has_cpucfg cpu_opt(LOONGARCH_CPU_CPUCFG) +#define cpu_has_lam cpu_opt(LOONGARCH_CPU_LAM) +#define cpu_has_ual cpu_opt(LOONGARCH_CPU_UAL) +#define cpu_has_fpu cpu_opt(LOONGARCH_CPU_FPU) +#define cpu_has_lsx cpu_opt(LOONGARCH_CPU_LSX) +#define cpu_has_lasx cpu_opt(LOONGARCH_CPU_LASX) +#define cpu_has_crc32 cpu_opt(LOONGARCH_CPU_CRC32) +#define cpu_has_complex cpu_opt(LOONGARCH_CPU_COMPLEX) +#define cpu_has_crypto cpu_opt(LOONGARCH_CPU_CRYPTO) +#define cpu_has_lvz cpu_opt(LOONGARCH_CPU_LVZ) +#define cpu_has_lbt_x86 cpu_opt(LOONGARCH_CPU_LBT_X86) +#define cpu_has_lbt_arm cpu_opt(LOONGARCH_CPU_LBT_ARM) +#define cpu_has_lbt_mips cpu_opt(LOONGARCH_CPU_LBT_MIPS) +#define cpu_has_lbt (cpu_has_lbt_x86|cpu_has_lbt_arm|cpu_has_lbt_mips) +#define cpu_has_csr cpu_opt(LOONGARCH_CPU_CSR) +#define cpu_has_tlb cpu_opt(LOONGARCH_CPU_TLB) +#define cpu_has_watch cpu_opt(LOONGARCH_CPU_WATCH) +#define cpu_has_vint cpu_opt(LOONGARCH_CPU_VINT) +#define cpu_has_csripi cpu_opt(LOONGARCH_CPU_CSRIPI) +#define cpu_has_extioi cpu_opt(LOONGARCH_CPU_EXTIOI) +#define cpu_has_prefetch cpu_opt(LOONGARCH_CPU_PREFETCH) +#define cpu_has_pmp cpu_opt(LOONGARCH_CPU_PMP) +#define cpu_has_perf cpu_opt(LOONGARCH_CPU_PMP) +#define cpu_has_scalefreq cpu_opt(LOONGARCH_CPU_SCALEFREQ) +#define cpu_has_flatmode cpu_opt(LOONGARCH_CPU_FLATMODE) +#define cpu_has_eiodecode cpu_opt(LOONGARCH_CPU_EIODECODE) +#define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) +#define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) +#define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW) + +#define cpu_has_matc_guest (cpu_data[0].guest_cfg & (1 << 0)) +#define cpu_has_matc_root (cpu_data[0].guest_cfg & (1 << 1)) +#define cpu_has_matc_nest (cpu_data[0].guest_cfg & (1 << 2)) +#define cpu_has_sitp (cpu_data[0].guest_cfg & (1 << 6)) +#define cpu_has_titp (cpu_data[0].guest_cfg & (1 << 8)) +#define cpu_has_toep (cpu_data[0].guest_cfg & (1 << 10)) +#define cpu_has_topp (cpu_data[0].guest_cfg & (1 << 12)) +#define cpu_has_torup (cpu_data[0].guest_cfg & (1 << 14)) +#define cpu_has_gcip_all (cpu_data[0].guest_cfg & (1 << 16)) +#define cpu_has_gcip_hit (cpu_data[0].guest_cfg & (1 << 17)) +#define cpu_has_gcip_secure (cpu_data[0].guest_cfg & (1 << 18)) + +/* + * Guest capabilities + */ +#define cpu_guest_has_conf1 (cpu_data[0].guest.conf & (1 << 1)) +#define cpu_guest_has_conf2 (cpu_data[0].guest.conf & (1 << 2)) +#define cpu_guest_has_conf3 (cpu_data[0].guest.conf & (1 << 3)) +#define cpu_guest_has_fpu (cpu_data[0].guest.options & LOONGARCH_CPU_FPU) +#define cpu_guest_has_perf (cpu_data[0].guest.options & LOONGARCH_CPU_PMP) +#define cpu_guest_has_watch (cpu_data[0].guest.options & LOONGARCH_CPU_WATCH) +#define cpu_guest_has_lsx (cpu_data[0].guest.ases & LOONGARCH_ASE_LSX) +#define cpu_guest_has_kscr(n) (cpu_data[0].guest.kscratch_mask & (1u << (n))) + +/* + * Guest dynamic capabilities + */ +#define cpu_guest_has_dyn_fpu (cpu_data[0].guest.options_dyn & LOONGARCH_CPU_FPU) +#define cpu_guest_has_dyn_perf (cpu_data[0].guest.options_dyn & LOONGARCH_CPU_PMP) +#define cpu_guest_has_dyn_lsx (cpu_data[0].guest.ases_dyn & LOONGARCH_ASE_LSX) + +#endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/loongarch/include/asm/cpu-info.h b/arch/loongarch/include/asm/cpu-info.h new file mode 100644 index 0000000000000000000000000000000000000000..2e3bd40c5e4d197a0477ec32aad3633aea9d2f85 --- /dev/null +++ b/arch/loongarch/include/asm/cpu-info.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_CPU_INFO_H +#define __ASM_CPU_INFO_H + +#include +#include + +#include + +/* cache_desc->flags */ +enum { + CACHE_PRESENT = (1 << 0), + CACHE_PRIVATE = (1 << 1), /* core private cache */ + CACHE_INCLUSIVE = (1 << 2), /* include the inner level caches */ +}; + +/* + * Descriptor for a cache + */ +struct cache_desc { + unsigned char type; + unsigned char level; + unsigned short sets; /* Number of lines per set */ + unsigned char ways; /* Number of ways */ + unsigned char linesz; /* Size of line in bytes */ + unsigned char flags; /* Flags describing cache properties */ +}; + +#define CACHE_LEVEL_MAX 3 +#define CACHE_LEAVES_MAX 6 + +struct guest_info { + unsigned long ases; + unsigned long ases_dyn; + unsigned long long options; + unsigned long long options_dyn; + u8 conf; + unsigned int kscratch_mask; +}; + +struct cpuinfo_loongarch { + u64 asid_cache; + unsigned long asid_mask; + + /* + * Capability and feature descriptor structure for LoongArch CPU + */ + unsigned long ases; + unsigned long long options; + unsigned int processor_id; + unsigned int fpu_vers; + unsigned int fpu_csr0; + unsigned int fpu_mask; + unsigned int cputype; + int isa_level; + int tlbsize; + int tlbsizemtlb; + int tlbsizestlbsets; + int tlbsizestlbways; + int cache_leaves_present; /* number of cache_leaves[] elements */ + struct cache_desc cache_leaves[CACHE_LEAVES_MAX]; + int core; /* physical core number in package */ + int package;/* physical package number */ + int global_id; /* physical global thread number */ + int vabits; /* Virtual Address size in bits */ + int pabits; /* Physical Address size in bits */ + unsigned int ksave_mask; /* Usable KSave mask. */ + unsigned int watch_dreg_count; /* Number data breakpoints */ + unsigned int watch_ireg_count; /* Number instruction breakpoints */ + unsigned int watch_reg_use_cnt; /* min(NUM_WATCH_REGS, watch_dreg_count + watch_ireg_count), Usable by ptrace */ + + /* VZ & Guest features */ + struct guest_info guest; + unsigned long guest_cfg; +} __attribute__((aligned(SMP_CACHE_BYTES))); + +extern struct cpuinfo_loongarch cpu_data[]; +#define boot_cpu_data cpu_data[0] +#define current_cpu_data cpu_data[smp_processor_id()] +#define raw_current_cpu_data cpu_data[raw_smp_processor_id()] + +extern void cpu_probe(void); + +extern const char *__cpu_family[]; +extern const char *__cpu_full_name[]; +#define cpu_family_string() __cpu_family[raw_smp_processor_id()] +#define cpu_full_name_string() __cpu_full_name[raw_smp_processor_id()] + +struct seq_file; +struct notifier_block; + +extern int register_proc_cpuinfo_notifier(struct notifier_block *nb); +extern int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v); + +#define proc_cpuinfo_notifier(fn, pri) \ +({ \ + static struct notifier_block fn##_nb = { \ + .notifier_call = fn, \ + .priority = pri \ + }; \ + \ + register_proc_cpuinfo_notifier(&fn##_nb); \ +}) + +struct proc_cpuinfo_notifier_args { + struct seq_file *m; + unsigned long n; +}; + +static inline bool cpus_are_siblings(int cpua, int cpub) +{ + struct cpuinfo_loongarch *infoa = &cpu_data[cpua]; + struct cpuinfo_loongarch *infob = &cpu_data[cpub]; + + if (infoa->package != infob->package) + return false; + + if (infoa->core != infob->core) + return false; + + return true; +} + +static inline unsigned long cpu_asid_mask(struct cpuinfo_loongarch *cpuinfo) +{ + return cpuinfo->asid_mask; +} + +static inline void set_cpu_asid_mask(struct cpuinfo_loongarch *cpuinfo, + unsigned long asid_mask) +{ + cpuinfo->asid_mask = asid_mask; +} + +#endif /* __ASM_CPU_INFO_H */ diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..5971faec6d5ac45bb241d76146f60cb3e3fd5abd --- /dev/null +++ b/arch/loongarch/include/asm/cpu.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * cpu.h: Values of the PRID register used to match up + * various LoongArch cpu types. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_CPU_H +#define _ASM_CPU_H + +/* + * As described in LoongArch specs from Loongson Technology, the PRID register + * (CPUCFG.00) has the following layout: + * + * +---------------+----------------+------------+--------------------+ + * | Reserved | Company ID | Series ID | Product ID | + * +---------------+----------------+------------+--------------------+ + * 31 24 23 16 15 12 11 0 + */ + +/* + * Assigned Company values for bits 23:16 of the PRID register. + */ + +#define PRID_COMP_MASK 0xff0000 + +#define PRID_COMP_LOONGSON 0x140000 + +/* + * Assigned Series ID values for bits 15:12 of the PRID register. In order + * to detect a certain CPU type exactly eventually additional registers may + * need to be examined. + */ + +#define PRID_SERIES_MASK 0xf000 + +#define PRID_SERIES_LA132 0x8000 /* Loongson 32bit */ +#define PRID_SERIES_LA264 0xa000 /* Loongson 64bit, 2-issue */ +#define PRID_SERIES_LA364 0xb000 /* Loongson 64bit, 3-issue */ +#define PRID_SERIES_LA464 0xc000 /* Loongson 64bit, 4-issue */ +#define PRID_SERIES_LA664 0xd000 /* Loongson 64bit, 6-issue */ + +/* + * Particular Product ID values for bits 11:0 of the PRID register. + */ + +#define PRID_PRODUCT_MASK 0x0fff + +#if !defined(__ASSEMBLY__) + +enum cpu_type_enum { + CPU_UNKNOWN, + CPU_LOONGSON32, + CPU_LOONGSON64, + CPU_LAST +}; + +#endif /* !__ASSEMBLY */ + +/* + * ISA Level encodings + * + */ + +#define LOONGARCH_CPU_ISA_LA32R 0x00000001 +#define LOONGARCH_CPU_ISA_LA32S 0x00000002 +#define LOONGARCH_CPU_ISA_LA64 0x00000004 + +#define LOONGARCH_CPU_ISA_32BIT (LOONGARCH_CPU_ISA_LA32R | LOONGARCH_CPU_ISA_LA32S) +#define LOONGARCH_CPU_ISA_64BIT LOONGARCH_CPU_ISA_LA64 + +/* + * CPU Option encodings + */ +#define CPU_FEATURE_CPUCFG 0 /* CPU has CPUCFG */ +#define CPU_FEATURE_LAM 1 /* CPU has Atomic instructions */ +#define CPU_FEATURE_UAL 2 /* CPU supports unaligned access */ +#define CPU_FEATURE_FPU 3 /* CPU has FPU */ +#define CPU_FEATURE_LSX 4 /* CPU has LSX (128-bit SIMD) */ +#define CPU_FEATURE_LASX 5 /* CPU has LASX (256-bit SIMD) */ +#define CPU_FEATURE_CRC32 6 /* CPU has CRC32 instructions */ +#define CPU_FEATURE_COMPLEX 7 /* CPU has Complex instructions */ +#define CPU_FEATURE_CRYPTO 8 /* CPU has Crypto instructions */ +#define CPU_FEATURE_LVZ 9 /* CPU has Virtualization extension */ +#define CPU_FEATURE_LBT_X86 10 /* CPU has X86 Binary Translation */ +#define CPU_FEATURE_LBT_ARM 11 /* CPU has ARM Binary Translation */ +#define CPU_FEATURE_LBT_MIPS 12 /* CPU has MIPS Binary Translation */ +#define CPU_FEATURE_TLB 13 /* CPU has TLB */ +#define CPU_FEATURE_CSR 14 /* CPU has CSR */ +#define CPU_FEATURE_WATCH 15 /* CPU has watchpoint registers */ +#define CPU_FEATURE_VINT 16 /* CPU has vectored interrupts */ +#define CPU_FEATURE_CSRIPI 17 /* CPU has CSR-IPI */ +#define CPU_FEATURE_EXTIOI 18 /* CPU has EXT-IOI */ +#define CPU_FEATURE_PREFETCH 19 /* CPU has prefetch instructions */ +#define CPU_FEATURE_PMP 20 /* CPU has perfermance counter */ +#define CPU_FEATURE_SCALEFREQ 21 /* CPU supports cpufreq scaling */ +#define CPU_FEATURE_FLATMODE 22 /* CPU has flat mode */ +#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */ +#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ +#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ +#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */ + +#define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) +#define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) +#define LOONGARCH_CPU_UAL BIT_ULL(CPU_FEATURE_UAL) +#define LOONGARCH_CPU_FPU BIT_ULL(CPU_FEATURE_FPU) +#define LOONGARCH_CPU_LSX BIT_ULL(CPU_FEATURE_LSX) +#define LOONGARCH_CPU_LASX BIT_ULL(CPU_FEATURE_LASX) +#define LOONGARCH_CPU_CRC32 BIT_ULL(CPU_FEATURE_CRC32) +#define LOONGARCH_CPU_COMPLEX BIT_ULL(CPU_FEATURE_COMPLEX) +#define LOONGARCH_CPU_CRYPTO BIT_ULL(CPU_FEATURE_CRYPTO) +#define LOONGARCH_CPU_LVZ BIT_ULL(CPU_FEATURE_LVZ) +#define LOONGARCH_CPU_LBT_X86 BIT_ULL(CPU_FEATURE_LBT_X86) +#define LOONGARCH_CPU_LBT_ARM BIT_ULL(CPU_FEATURE_LBT_ARM) +#define LOONGARCH_CPU_LBT_MIPS BIT_ULL(CPU_FEATURE_LBT_MIPS) +#define LOONGARCH_CPU_TLB BIT_ULL(CPU_FEATURE_TLB) +#define LOONGARCH_CPU_CSR BIT_ULL(CPU_FEATURE_CSR) +#define LOONGARCH_CPU_WATCH BIT_ULL(CPU_FEATURE_WATCH) +#define LOONGARCH_CPU_VINT BIT_ULL(CPU_FEATURE_VINT) +#define LOONGARCH_CPU_CSRIPI BIT_ULL(CPU_FEATURE_CSRIPI) +#define LOONGARCH_CPU_EXTIOI BIT_ULL(CPU_FEATURE_EXTIOI) +#define LOONGARCH_CPU_PREFETCH BIT_ULL(CPU_FEATURE_PREFETCH) +#define LOONGARCH_CPU_PMP BIT_ULL(CPU_FEATURE_PMP) +#define LOONGARCH_CPU_SCALEFREQ BIT_ULL(CPU_FEATURE_SCALEFREQ) +#define LOONGARCH_CPU_FLATMODE BIT_ULL(CPU_FEATURE_FLATMODE) +#define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE) +#define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) +#define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) +#define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW) +#endif /* _ASM_CPU_H */ diff --git a/arch/loongarch/include/asm/cpufeature.h b/arch/loongarch/include/asm/cpufeature.h new file mode 100644 index 0000000000000000000000000000000000000000..8cf9cc9cd072403203611e05c63f9f75579e3282 --- /dev/null +++ b/arch/loongarch/include/asm/cpufeature.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * CPU feature definitions for module loading, used by + * module_cpu_feature_match(), see uapi/asm/hwcap.h for LoongArch CPU features. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#ifndef __ASM_CPUFEATURE_H +#define __ASM_CPUFEATURE_H + +#include +#include + +#define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) + +#define cpu_feature(x) ilog2(HWCAP_ ## x) + +static inline bool cpu_have_feature(unsigned int num) +{ + return elf_hwcap & (1UL << num); +} + +#endif /* __ASM_CPUFEATURE_H */ diff --git a/arch/loongarch/include/asm/delay.h b/arch/loongarch/include/asm/delay.h new file mode 100644 index 0000000000000000000000000000000000000000..9536bcc3cfaeb1c97fe9563a396843aadd926b1a --- /dev/null +++ b/arch/loongarch/include/asm/delay.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994 by Waldorf Electronics + * Copyright (C) 1995 - 2000, 01, 03 by Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2007 Maciej W. Rozycki + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_DELAY_H +#define _ASM_DELAY_H + +#include + +extern void __delay(unsigned long cycles); +extern void __ndelay(unsigned long ns); +extern void __udelay(unsigned long us); + +#define ndelay(ns) __ndelay(ns) +#define udelay(us) __udelay(us) + +/* make sure "usecs *= ..." in udelay do not overflow. */ +#if HZ >= 1000 +#define MAX_UDELAY_MS 1 +#elif HZ <= 200 +#define MAX_UDELAY_MS 5 +#else +#define MAX_UDELAY_MS (1000 / HZ) +#endif + +#endif /* _ASM_DELAY_H */ diff --git a/arch/loongarch/include/asm/dma-direct.h b/arch/loongarch/include/asm/dma-direct.h new file mode 100644 index 0000000000000000000000000000000000000000..7056788e9c1e56f60b485976774fa12293b55343 --- /dev/null +++ b/arch/loongarch/include/asm/dma-direct.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + */ +#ifndef _LOONGARCH_DMA_DIRECT_H +#define _LOONGARCH_DMA_DIRECT_H + +dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); +phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); + +#endif /* _LOONGARCH_DMA_DIRECT_H */ diff --git a/arch/loongarch/include/asm/dma.h b/arch/loongarch/include/asm/dma.h new file mode 100644 index 0000000000000000000000000000000000000000..7e5b3e7d462b637edb7205e0cd04bfd0d144c4ca --- /dev/null +++ b/arch/loongarch/include/asm/dma.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * + */ +#ifndef __ASM_DMA_H +#define __ASM_DMA_H + +#define MAX_DMA_ADDRESS PAGE_OFFSET +#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) + +extern int isa_dma_bridge_buggy; + +#endif diff --git a/arch/loongarch/include/asm/dmi.h b/arch/loongarch/include/asm/dmi.h new file mode 100644 index 0000000000000000000000000000000000000000..cdc29017c3fd6b304ac42e0efba1660a9f7718b5 --- /dev/null +++ b/arch/loongarch/include/asm/dmi.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_DMI_H +#define _ASM_DMI_H + +#include +#include + +#define dmi_early_remap(x, l) dmi_remap(x, l) +#define dmi_early_unmap(x, l) dmi_unmap(x) +#define dmi_alloc(l) memblock_alloc(l, PAGE_SIZE) + +static inline void *dmi_remap(u64 phys_addr, unsigned long size) +{ + return ((void *)TO_CACHE(phys_addr)); +} + +static inline void dmi_unmap(void *addr) +{ +} + +#endif /* _ASM_DMI_H */ diff --git a/arch/loongarch/include/asm/efi.h b/arch/loongarch/include/asm/efi.h new file mode 100644 index 0000000000000000000000000000000000000000..5ae447c4b21e326cae9b31c2cdc931f81a3bf05f --- /dev/null +++ b/arch/loongarch/include/asm/efi.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_LOONGARCH_EFI_H +#define _ASM_LOONGARCH_EFI_H + +#include + +void __init efi_init(void); +void __init efi_runtime_init(void); + +#define ARCH_EFI_IRQ_FLAGS_MASK 0x00000004 /* Bit 2: CSR.CRMD.IE */ + +static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) +{ +} + +#define arch_efi_call_virt_setup() \ +({ \ +}) + +#define arch_efi_call_virt(p, f, args...) \ +({ \ + efi_##f##_t * __f; \ + __f = p->f; \ + __f(args); \ +}) + +#define arch_efi_call_virt_teardown() \ +({ \ +}) + +#define EFI_ALLOC_ALIGN SZ_64K +#define EFI_RT_VIRTUAL_OFFSET CSR_DMW0_BASE + +struct screen_info *alloc_screen_info(void); +void free_screen_info(struct screen_info *si); + +static inline unsigned long efi_get_max_fdt_addr(unsigned long image_addr) +{ + return ULONG_MAX; +} + +static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) +{ + return ULONG_MAX; +} + +#endif /* _ASM_LOONGARCH_EFI_H */ diff --git a/arch/loongarch/include/asm/elf.h b/arch/loongarch/include/asm/elf.h new file mode 100644 index 0000000000000000000000000000000000000000..2715cfc10b30cd8a019588e1dd11d9cd30afa8e8 --- /dev/null +++ b/arch/loongarch/include/asm/elf.h @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Much of this is taken from binutils and GNU libc ... + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_ELF_H +#define _ASM_ELF_H + +#include +#include +#include + +#include +#include + +/* The ABI of a file. */ +#define EF_LOONGARCH_ABI_LP64_SOFT_FLOAT 0x1 +#define EF_LOONGARCH_ABI_LP64_SINGLE_FLOAT 0x2 +#define EF_LOONGARCH_ABI_LP64_DOUBLE_FLOAT 0x3 + +#define EF_LOONGARCH_ABI_ILP32_SOFT_FLOAT 0x5 +#define EF_LOONGARCH_ABI_ILP32_SINGLE_FLOAT 0x6 +#define EF_LOONGARCH_ABI_ILP32_DOUBLE_FLOAT 0x7 + +/* LoongArch relocation types used by the dynamic linker */ +#define R_LARCH_NONE 0 +#define R_LARCH_32 1 +#define R_LARCH_64 2 +#define R_LARCH_RELATIVE 3 +#define R_LARCH_COPY 4 +#define R_LARCH_JUMP_SLOT 5 +#define R_LARCH_TLS_DTPMOD32 6 +#define R_LARCH_TLS_DTPMOD64 7 +#define R_LARCH_TLS_DTPREL32 8 +#define R_LARCH_TLS_DTPREL64 9 +#define R_LARCH_TLS_TPREL32 10 +#define R_LARCH_TLS_TPREL64 11 +#define R_LARCH_IRELATIVE 12 +#define R_LARCH_MARK_LA 20 +#define R_LARCH_MARK_PCREL 21 +#define R_LARCH_SOP_PUSH_PCREL 22 +#define R_LARCH_SOP_PUSH_ABSOLUTE 23 +#define R_LARCH_SOP_PUSH_DUP 24 +#define R_LARCH_SOP_PUSH_GPREL 25 +#define R_LARCH_SOP_PUSH_TLS_TPREL 26 +#define R_LARCH_SOP_PUSH_TLS_GOT 27 +#define R_LARCH_SOP_PUSH_TLS_GD 28 +#define R_LARCH_SOP_PUSH_PLT_PCREL 29 +#define R_LARCH_SOP_ASSERT 30 +#define R_LARCH_SOP_NOT 31 +#define R_LARCH_SOP_SUB 32 +#define R_LARCH_SOP_SL 33 +#define R_LARCH_SOP_SR 34 +#define R_LARCH_SOP_ADD 35 +#define R_LARCH_SOP_AND 36 +#define R_LARCH_SOP_IF_ELSE 37 +#define R_LARCH_SOP_POP_32_S_10_5 38 +#define R_LARCH_SOP_POP_32_U_10_12 39 +#define R_LARCH_SOP_POP_32_S_10_12 40 +#define R_LARCH_SOP_POP_32_S_10_16 41 +#define R_LARCH_SOP_POP_32_S_10_16_S2 42 +#define R_LARCH_SOP_POP_32_S_5_20 43 +#define R_LARCH_SOP_POP_32_S_0_5_10_16_S2 44 +#define R_LARCH_SOP_POP_32_S_0_10_10_16_S2 45 +#define R_LARCH_SOP_POP_32_U 46 +#define R_LARCH_ADD8 47 +#define R_LARCH_ADD16 48 +#define R_LARCH_ADD24 49 +#define R_LARCH_ADD32 50 +#define R_LARCH_ADD64 51 +#define R_LARCH_SUB8 52 +#define R_LARCH_SUB16 53 +#define R_LARCH_SUB24 54 +#define R_LARCH_SUB32 55 +#define R_LARCH_SUB64 56 +#define R_LARCH_GNU_VTINHERIT 57 +#define R_LARCH_GNU_VTENTRY 58 +#define R_LARCH_B16 64 +#define R_LARCH_B21 65 +#define R_LARCH_B26 66 +#define R_LARCH_ABS_HI20 67 +#define R_LARCH_ABS_LO12 68 +#define R_LARCH_ABS64_LO20 69 +#define R_LARCH_ABS64_HI12 70 +#define R_LARCH_PCALA_HI20 71 +#define R_LARCH_PCALA_LO12 72 +#define R_LARCH_PCALA64_LO20 73 +#define R_LARCH_PCALA64_HI12 74 +#define R_LARCH_GOT_PC_HI20 75 +#define R_LARCH_GOT_PC_LO12 76 +#define R_LARCH_GOT64_PC_LO20 77 +#define R_LARCH_GOT64_PC_HI12 78 +#define R_LARCH_GOT_HI20 79 +#define R_LARCH_GOT_LO12 80 +#define R_LARCH_GOT64_LO20 81 +#define R_LARCH_GOT64_HI12 82 +#define R_LARCH_TLS_LE_HI20 83 +#define R_LARCH_TLS_LE_LO12 84 +#define R_LARCH_TLS_LE64_LO20 85 +#define R_LARCH_TLS_LE64_HI12 86 +#define R_LARCH_TLS_IE_PC_HI20 87 +#define R_LARCH_TLS_IE_PC_LO12 88 +#define R_LARCH_TLS_IE64_PC_LO20 89 +#define R_LARCH_TLS_IE64_PC_HI12 90 +#define R_LARCH_TLS_IE_HI20 91 +#define R_LARCH_TLS_IE_LO12 92 +#define R_LARCH_TLS_IE64_LO20 93 +#define R_LARCH_TLS_IE64_HI12 94 +#define R_LARCH_TLS_LD_PC_HI20 95 +#define R_LARCH_TLS_LD_HI20 96 +#define R_LARCH_TLS_GD_PC_HI20 97 +#define R_LARCH_TLS_GD_HI20 98 +#define R_LARCH_32_PCREL 99 +#define R_LARCH_RELAX 100 +#define R_LARCH_DELETE 101 +#define R_LARCH_ALIGN 102 +#define R_LARCH_PCREL20_S2 103 +#define R_LARCH_CFA 104 +#define R_LARCH_ADD6 105 +#define R_LARCH_SUB6 106 +#define R_LARCH_ADD_ULEB128 107 +#define R_LARCH_SUB_ULEB128 108 +#define R_LARCH_64_PCREL 109 + +#ifndef ELF_ARCH + +/* ELF register definitions */ + +/* + * General purpose have the following registers: + * Register Number + * GPRs 32 + * ORIG_A0 1 + * ERA 1 + * BADVADDR 1 + * CRMD 1 + * PRMD 1 + * EUEN 1 + * ECFG 1 + * ESTAT 1 + * Reserved 5 + */ +#define ELF_NGREG 45 + +/* + * Floating point have the following registers: + * Register Number + * FPR 32 + * FCC 1 + * FCSR 1 + */ +#define ELF_NFPREG 34 + +typedef unsigned long elf_greg_t; +typedef elf_greg_t elf_gregset_t[ELF_NGREG]; + +typedef double elf_fpreg_t; +typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; + +void loongarch_dump_regs64(u64 *uregs, const struct pt_regs *regs); + +#ifdef CONFIG_32BIT +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch elf32_check_arch + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_CLASS ELFCLASS32 + +#define ELF_CORE_COPY_REGS(dest, regs) \ + loongarch_dump_regs32((u32 *)&(dest), (regs)); + +#endif /* CONFIG_32BIT */ + +#ifdef CONFIG_64BIT +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch elf64_check_arch + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_CLASS ELFCLASS64 + +#define ELF_CORE_COPY_REGS(dest, regs) \ + loongarch_dump_regs64((u64 *)&(dest), (regs)); + +#endif /* CONFIG_64BIT */ + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_DATA ELFDATA2LSB +#define ELF_ARCH EM_LOONGARCH + +#endif /* !defined(ELF_ARCH) */ + +#define loongarch_elf_check_machine(x) ((x)->e_machine == EM_LOONGARCH) + +#define vmcore_elf32_check_arch loongarch_elf_check_machine +#define vmcore_elf64_check_arch loongarch_elf_check_machine + +/* + * Return non-zero if HDR identifies an 32bit ELF binary. + */ +#define elf32_check_arch(hdr) \ +({ \ + int __res = 1; \ + struct elfhdr *__h = (hdr); \ + \ + if (!loongarch_elf_check_machine(__h)) \ + __res = 0; \ + if (__h->e_ident[EI_CLASS] != ELFCLASS32) \ + __res = 0; \ + \ + __res; \ +}) + +/* + * Return non-zero if HDR identifies an 64bit ELF binary. + */ +#define elf64_check_arch(hdr) \ +({ \ + int __res = 1; \ + struct elfhdr *__h = (hdr); \ + \ + if (!loongarch_elf_check_machine(__h)) \ + __res = 0; \ + if (__h->e_ident[EI_CLASS] != ELFCLASS64) \ + __res = 0; \ + \ + __res; \ +}) + +#ifdef CONFIG_32BIT + +#define SET_PERSONALITY2(ex, state) \ +do { \ + current->thread.vdso = &vdso_info; \ + \ + if (personality(current->personality) != PER_LINUX) \ + set_personality(PER_LINUX); \ +} while (0) + +#endif /* CONFIG_32BIT */ + +#ifdef CONFIG_64BIT + +#define SET_PERSONALITY2(ex, state) \ +do { \ + unsigned int p; \ + \ + clear_thread_flag(TIF_32BIT_REGS); \ + clear_thread_flag(TIF_32BIT_ADDR); \ + \ + current->thread.vdso = &vdso_info; \ + \ + p = personality(current->personality); \ + if (p != PER_LINUX32 && p != PER_LINUX) \ + set_personality(PER_LINUX); \ +} while (0) + +#endif /* CONFIG_64BIT */ + +#define CORE_DUMP_USE_REGSET +#define ELF_EXEC_PAGESIZE PAGE_SIZE + +/* This yields a mask that user programs can use to figure out what + instruction set this cpu supports. This could be done in userspace, + but it's not easy, and we've already done it here. */ + +#define ELF_HWCAP (elf_hwcap) +extern unsigned int elf_hwcap; +#include + +/* + * This yields a string that ld.so will use to load implementation + * specific libraries for optimization. This is more specific in + * intent than poking at uname or /proc/cpuinfo. + */ + +#define ELF_PLATFORM __elf_platform +extern const char *__elf_platform; + +#define ELF_PLAT_INIT(_r, load_addr) do { \ + _r->regs[1] = _r->regs[2] = _r->regs[3] = _r->regs[4] = 0; \ + _r->regs[5] = _r->regs[6] = _r->regs[7] = _r->regs[8] = 0; \ + _r->regs[9] = _r->regs[10] /* syscall n */ = _r->regs[12] = 0; \ + _r->regs[13] = _r->regs[14] = _r->regs[15] = _r->regs[16] = 0; \ + _r->regs[17] = _r->regs[18] = _r->regs[19] = _r->regs[20] = 0; \ + _r->regs[21] = _r->regs[22] = _r->regs[23] = _r->regs[24] = 0; \ + _r->regs[25] = _r->regs[26] = _r->regs[27] = _r->regs[28] = 0; \ + _r->regs[29] = _r->regs[30] = _r->regs[31] = 0; \ +} while (0) + +/* This is the location that an ET_DYN program is loaded if exec'ed. Typical + use of this is to invoke "./ld.so someprog" to test out a new version of + the loader. We need to make sure that it is out of the way of the program + that it will "exec", and that there is sufficient room for the brk. */ + +#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2) + +/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ +#define ARCH_DLINFO \ +do { \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (unsigned long)current->mm->context.vdso); \ +} while (0) + +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +struct linux_binprm; +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp); + +struct arch_elf_state { + int fp_abi; + int interp_fp_abi; +}; + +#define LOONGARCH_ABI_FP_ANY (0) + +#define INIT_ARCH_ELF_STATE { \ + .fp_abi = LOONGARCH_ABI_FP_ANY, \ + .interp_fp_abi = LOONGARCH_ABI_FP_ANY, \ +} + +#define elf_read_implies_exec(ex, exec_stk) (exec_stk == EXSTACK_DEFAULT) + +extern int arch_elf_pt_proc(void *ehdr, void *phdr, struct file *elf, + bool is_interp, struct arch_elf_state *state); + +extern int arch_check_elf(void *ehdr, bool has_interpreter, void *interp_ehdr, + struct arch_elf_state *state); + +#endif /* _ASM_ELF_H */ diff --git a/arch/loongarch/include/asm/entry-common.h b/arch/loongarch/include/asm/entry-common.h new file mode 100644 index 0000000000000000000000000000000000000000..0fe2a098ded96caec4e556d53552392a9fba2e60 --- /dev/null +++ b/arch/loongarch/include/asm/entry-common.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_LOONGARCH_ENTRY_COMMON_H +#define ARCH_LOONGARCH_ENTRY_COMMON_H + +#include +#include + +static inline bool on_thread_stack(void) +{ + return !(((unsigned long)(current->stack) ^ current_stack_pointer) & ~(THREAD_SIZE - 1)); +} + +#endif diff --git a/arch/loongarch/include/asm/errno.h b/arch/loongarch/include/asm/errno.h new file mode 100644 index 0000000000000000000000000000000000000000..2039ecb21cefa7310bef1daf25ecb1ae7da6f632 --- /dev/null +++ b/arch/loongarch/include/asm/errno.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1995, 1999, 2001, 2002 by Ralf Baechle + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_ERRNO_H +#define _ASM_ERRNO_H + +#include + + +/* The biggest error number defined here or in . */ +#define EMAXERRNO 1133 + +#endif /* _ASM_ERRNO_H */ diff --git a/arch/loongarch/include/asm/exception.h b/arch/loongarch/include/asm/exception.h new file mode 100644 index 0000000000000000000000000000000000000000..af74a3fdcad179f7582d0234c5cfd4cdca6d5d65 --- /dev/null +++ b/arch/loongarch/include/asm/exception.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_EXCEPTION_H +#define __ASM_EXCEPTION_H + +#include +#include + +void show_registers(struct pt_regs *regs); + +asmlinkage void cache_parity_error(void); +asmlinkage void noinstr do_ade(struct pt_regs *regs); +asmlinkage void noinstr do_ale(struct pt_regs *regs); +asmlinkage void noinstr do_bce(struct pt_regs *regs); +asmlinkage void noinstr do_bp(struct pt_regs *regs); +asmlinkage void noinstr do_ri(struct pt_regs *regs); +asmlinkage void noinstr do_fpu(struct pt_regs *regs); +asmlinkage void noinstr do_fpe(struct pt_regs *regs, unsigned long fcsr); +asmlinkage void noinstr do_lsx(struct pt_regs *regs); +asmlinkage void noinstr do_lasx(struct pt_regs *regs); +asmlinkage void noinstr do_lbt(struct pt_regs *regs); +asmlinkage void noinstr do_watch(struct pt_regs *regs); +asmlinkage void noinstr do_syscall(struct pt_regs *regs); +asmlinkage void noinstr do_reserved(struct pt_regs *regs); +asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp); +asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, + unsigned long write, unsigned long address); + +asmlinkage void handle_ade(void); +asmlinkage void handle_ale(void); +asmlinkage void handle_bce(void); +asmlinkage void handle_sys(void); +asmlinkage void handle_bp(void); +asmlinkage void handle_ri(void); +asmlinkage void handle_fpu(void); +asmlinkage void handle_fpe(void); +asmlinkage void handle_lsx(void); +asmlinkage void handle_lasx(void); +asmlinkage void handle_lbt(void); +asmlinkage void handle_watch(void); +asmlinkage void handle_reserved(void); +asmlinkage void handle_vint(void); +asmlinkage void noinstr handle_loongarch_irq(struct pt_regs *regs); + +#endif /* __ASM_EXCEPTION_H */ diff --git a/arch/loongarch/include/asm/exec.h b/arch/loongarch/include/asm/exec.h new file mode 100644 index 0000000000000000000000000000000000000000..371d5fb327a818c4cfaaa8293ac5e1c5a9dfb271 --- /dev/null +++ b/arch/loongarch/include/asm/exec.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994, 95, 96, 97, 98, 99, 2003, 06 by Ralf Baechle + * Copyright (C) 1996 by Paul M. Antoine + * Copyright (C) 1999 Silicon Graphics + * Kevin D. Kissell, kevink@mips.org and Carsten Langgaard, carstenl@mips.com + * Copyright (C) 2000 MIPS Technologies, Inc. + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_EXEC_H +#define _ASM_EXEC_H + +extern unsigned long arch_align_stack(unsigned long sp); + +#endif /* _ASM_EXEC_H */ diff --git a/arch/loongarch/include/asm/extable.h b/arch/loongarch/include/asm/extable.h new file mode 100644 index 0000000000000000000000000000000000000000..92612b4364a1d46cd0fb994ed3ae30152835d843 --- /dev/null +++ b/arch/loongarch/include/asm/extable.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_EXTABLE_H +#define _ASM_LOONGARCH_EXTABLE_H + +/* + * The exception table consists of pairs of relative offsets: the first + * is the relative offset to an instruction that is allowed to fault, + * and the second is the relative offset at which the program should + * continue. No registers are modified, so it is entirely up to the + * continuation code to figure out what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + int insn, fixup; + short type, data; +}; + +#define ARCH_HAS_RELATIVE_EXTABLE + +#define swap_ex_entry_fixup(a, b, tmp, delta) \ +do { \ + (a)->fixup = (b)->fixup + (delta); \ + (b)->fixup = (tmp).fixup - (delta); \ + (a)->type = (b)->type; \ + (b)->type = (tmp).type; \ + (a)->data = (b)->data; \ + (b)->data = (tmp).data; \ +} while (0) + +bool fixup_exception(struct pt_regs *regs); + +#endif diff --git a/arch/loongarch/include/asm/fb.h b/arch/loongarch/include/asm/fb.h new file mode 100644 index 0000000000000000000000000000000000000000..5cd96907d913ec204aebc97b5690de33eb879076 --- /dev/null +++ b/arch/loongarch/include/asm/fb.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is released under the GPLv2 + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_FB_H_ +#define _ASM_FB_H_ + +#include +#include +#include + +static inline void fb_pgprotect(struct file *file, struct vm_area_struct *vma, + unsigned long off) +{ + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +} + +static inline int fb_is_primary_device(struct fb_info *info) +{ + return 0; +} + +#endif /* _ASM_FB_H_ */ diff --git a/arch/loongarch/include/asm/fixmap.h b/arch/loongarch/include/asm/fixmap.h new file mode 100644 index 0000000000000000000000000000000000000000..76b1f550f289cbfa56cb2c5591aebe47dc51bf53 --- /dev/null +++ b/arch/loongarch/include/asm/fixmap.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fixmap.h: compile-time virtual memory allocation + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#ifndef _ASM_FIXMAP_H +#define _ASM_FIXMAP_H + +#include +#ifdef CONFIG_HIGHMEM +#include +#include +#endif + +/* + * Here we define all the compile-time 'special' virtual + * addresses. The point is to have a constant address at + * compile time, but to set the physical address only + * in the boot process. We allocate these special addresses + * from the end of virtual memory (0xfffff000) backwards. + * Also this lets us do fail-safe vmalloc(), we + * can guarantee that these special addresses and + * vmalloc()-ed addresses never overlap. + * + * these 'compile-time allocated' memory buffers are + * fixed-size 4k pages. (or larger if used with an increment + * highger than 1) use fixmap_set(idx,phys) to associate + * physical memory with fixmap indices. + * + * TLB entries of such buffers will not be flushed across + * task switches. + */ + +#define NR_FIX_BTMAPS 64 + +/* + * on UP currently we will have no trace of the fixmap mechanizm, + * no page table allocations, etc. This might change in the + * future, say framebuffers for the console driver(s) could be + * fix-mapped? + */ +enum fixed_addresses { + FIX_HOLE, +#define FIX_N_COLOURS 8 + FIX_CMAP_BEGIN, + FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * 2), +#ifdef CONFIG_HIGHMEM + /* reserved pte's for temporary kernel mappings */ + FIX_KMAP_BEGIN = FIX_CMAP_END + 1, + FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, +#endif + FIX_EARLYCON_MEM_BASE, + __end_of_fixed_addresses +}; + +/* + * used by vmalloc.c. + * + * Leave one empty page between vmalloc'ed areas and + * the start of the fixmap, and leave one page empty + * at the top of mem.. + */ +#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) +#define FIXMAP_PAGE_IO PAGE_KERNEL_SUC + +extern void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags); + +#include + +/* + * Called from pgtable_init() + */ +extern void fixrange_init(unsigned long start, unsigned long end, + pgd_t *pgd_base); + + +#endif diff --git a/arch/loongarch/include/asm/fpregdef.h b/arch/loongarch/include/asm/fpregdef.h new file mode 100644 index 0000000000000000000000000000000000000000..a7324cdcd75573f91e82218475c2ac337e1bf308 --- /dev/null +++ b/arch/loongarch/include/asm/fpregdef.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Definitions for the FPU register names + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_FPREGDEF_H +#define _ASM_FPREGDEF_H + +#define fa0 $f0 /* argument registers, fa0/fa1 reused as fv0/fv1 for return value */ +#define fa1 $f1 +#define fa2 $f2 +#define fa3 $f3 +#define fa4 $f4 +#define fa5 $f5 +#define fa6 $f6 +#define fa7 $f7 +#define ft0 $f8 /* caller saved */ +#define ft1 $f9 +#define ft2 $f10 +#define ft3 $f11 +#define ft4 $f12 +#define ft5 $f13 +#define ft6 $f14 +#define ft7 $f15 +#define ft8 $f16 +#define ft9 $f17 +#define ft10 $f18 +#define ft11 $f19 +#define ft12 $f20 +#define ft13 $f21 +#define ft14 $f22 +#define ft15 $f23 +#define fs0 $f24 /* callee saved */ +#define fs1 $f25 +#define fs2 $f26 +#define fs3 $f27 +#define fs4 $f28 +#define fs5 $f29 +#define fs6 $f30 +#define fs7 $f31 + +/* + * Current binutils expects *GPRs* at FCSR position for the FCSR + * operation instructions, so define aliases for those used. + */ +#define fcsr0 $r0 +#define fcsr1 $r1 +#define fcsr2 $r2 +#define fcsr3 $r3 + +#endif /* _ASM_FPREGDEF_H */ diff --git a/arch/loongarch/include/asm/fpu.h b/arch/loongarch/include/asm/fpu.h new file mode 100644 index 0000000000000000000000000000000000000000..6dca64890074670fd6aab0c89fddce98324c3c3f --- /dev/null +++ b/arch/loongarch/include/asm/fpu.h @@ -0,0 +1,325 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_FPU_H +#define _ASM_FPU_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +struct sigcontext; + +extern void kernel_fpu_begin(void); +extern void kernel_fpu_end(void); + +extern void _init_fpu(unsigned int); +extern void _save_fp(struct loongarch_fpu *); +extern void _restore_fp(struct loongarch_fpu *); + +extern void _save_lsx(struct loongarch_fpu *fpu); +extern void _restore_lsx(struct loongarch_fpu *fpu); +extern void _init_lsx_upper(void); +extern void _restore_lsx_upper(struct loongarch_fpu *fpu); + +extern void _save_lasx(struct loongarch_fpu *fpu); +extern void _restore_lasx(struct loongarch_fpu *fpu); +extern void _init_lasx_upper(void); +extern void _restore_lasx_upper(struct loongarch_fpu *fpu); + +static inline void enable_lsx(void); +static inline void disable_lsx(void); +static inline void save_lsx(struct task_struct *t); +static inline void restore_lsx(struct task_struct *t); + +static inline void enable_lasx(void); +static inline void disable_lasx(void); +static inline void save_lasx(struct task_struct *t); +static inline void restore_lasx(struct task_struct *t); + +/* + * Mask the FCSR Cause bits according to the Enable bits, observing + * that Unimplemented is always enabled. + */ +static inline unsigned long mask_fcsr_x(unsigned long fcsr) +{ + return fcsr & ((fcsr & FPU_CSR_ALL_E) << + (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E))); +} + +static inline int is_fp_enabled(void) +{ + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_FPEN) ? + 1 : 0; +} + +static inline int is_lsx_enabled(void) +{ + if (!cpu_has_lsx) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ? + 1 : 0; +} + +static inline int is_lasx_enabled(void) +{ + if (!cpu_has_lasx) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ? + 1 : 0; +} + +static inline int is_simd_enabled(void) +{ + return is_lsx_enabled() | is_lasx_enabled(); +} + +#define enable_fpu() \ +do { \ + set_csr_euen(CSR_EUEN_FPEN); \ +} while (0) + +#define disable_fpu() \ +do { \ + clear_csr_euen(CSR_EUEN_FPEN); \ +} while (0) + +#define clear_fpu_owner() clear_thread_flag(TIF_USEDFPU) + +static inline int is_fpu_owner(void) +{ + return test_thread_flag(TIF_USEDFPU); +} + +static inline void __own_fpu(void) +{ + enable_fpu(); + set_thread_flag(TIF_USEDFPU); + KSTK_EUEN(current) |= CSR_EUEN_FPEN; +} + +static inline void own_fpu_inatomic(int restore) +{ + if (cpu_has_fpu && !is_fpu_owner()) { + __own_fpu(); + if (restore) + _restore_fp(¤t->thread.fpu); + } +} + +static inline void own_fpu(int restore) +{ + preempt_disable(); + own_fpu_inatomic(restore); + preempt_enable(); +} + +static inline void lose_fpu_inatomic(int save, struct task_struct *tsk) +{ + if (is_fpu_owner()) { + if (!is_simd_enabled()) { + if (save) + _save_fp(&tsk->thread.fpu); + disable_fpu(); + } else { + if (save) { + if (!is_lasx_enabled()) + save_lsx(tsk); + else + save_lasx(tsk); + } + disable_fpu(); + disable_lsx(); + disable_lasx(); + clear_tsk_thread_flag(tsk, TIF_USEDSIMD); + } + clear_tsk_thread_flag(tsk, TIF_USEDFPU); + } + KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); +} + +static inline void lose_fpu(int save) +{ + preempt_disable(); + lose_fpu_inatomic(save, current); + preempt_enable(); +} + +static inline void init_fpu(void) +{ + unsigned int fcsr = current->thread.fpu.fcsr; + + __own_fpu(); + _init_fpu(fcsr); + set_used_math(); +} + +static inline void save_fp(struct task_struct *tsk) +{ + if (cpu_has_fpu) + _save_fp(&tsk->thread.fpu); +} + +static inline void restore_fp(struct task_struct *tsk) +{ + if (cpu_has_fpu) + _restore_fp(&tsk->thread.fpu); +} + +static inline void save_fpu_regs(struct task_struct *tsk) +{ + unsigned int euen; + + if (tsk == current) { + preempt_disable(); + + euen = csr_read32(LOONGARCH_CSR_EUEN); + +#ifdef CONFIG_CPU_HAS_LASX + if (euen & CSR_EUEN_LASXEN) + _save_lasx(¤t->thread.fpu); + else +#endif +#ifdef CONFIG_CPU_HAS_LSX + if (euen & CSR_EUEN_LSXEN) + _save_lsx(¤t->thread.fpu); + else +#endif + if (euen & CSR_EUEN_FPEN) + _save_fp(¤t->thread.fpu); + + preempt_enable(); + } +} + +static inline int is_simd_owner(void) +{ + return test_thread_flag(TIF_USEDSIMD); +} + +#ifdef CONFIG_CPU_HAS_LSX + +static inline void enable_lsx(void) +{ + if (cpu_has_lsx) + csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lsx(void) +{ + if (cpu_has_lsx) + csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN); +} + +static inline void save_lsx(struct task_struct *t) +{ + if (cpu_has_lsx) + _save_lsx(&t->thread.fpu); +} + +static inline void restore_lsx(struct task_struct *t) +{ + if (cpu_has_lsx) + _restore_lsx(&t->thread.fpu); +} + +static inline void init_lsx_upper(void) +{ + if (cpu_has_lsx) + _init_lsx_upper(); +} + +static inline void restore_lsx_upper(struct task_struct *t) +{ + if (cpu_has_lsx) + _restore_lsx_upper(&t->thread.fpu); +} + +#else +static inline void enable_lsx(void) {} +static inline void disable_lsx(void) {} +static inline void save_lsx(struct task_struct *t) {} +static inline void restore_lsx(struct task_struct *t) {} +static inline void init_lsx_upper(void) {} +static inline void restore_lsx_upper(struct task_struct *t) {} +#endif + +#ifdef CONFIG_CPU_HAS_LASX + +static inline void enable_lasx(void) +{ + + if (cpu_has_lasx) + csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lasx(void) +{ + if (cpu_has_lasx) + csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN); +} + +static inline void save_lasx(struct task_struct *t) +{ + if (cpu_has_lasx) + _save_lasx(&t->thread.fpu); +} + +static inline void restore_lasx(struct task_struct *t) +{ + if (cpu_has_lasx) + _restore_lasx(&t->thread.fpu); +} + +static inline void init_lasx_upper(void) +{ + if (cpu_has_lasx) + _init_lasx_upper(); +} + +static inline void restore_lasx_upper(struct task_struct *t) +{ + if (cpu_has_lasx) + _restore_lasx_upper(&t->thread.fpu); +} + +#else +static inline void enable_lasx(void) {} +static inline void disable_lasx(void) {} +static inline void save_lasx(struct task_struct *t) {} +static inline void restore_lasx(struct task_struct *t) {} +static inline void init_lasx_upper(void) {} +static inline void restore_lasx_upper(struct task_struct *t) {} +#endif + +static inline int thread_lsx_context_live(void) +{ + if (!cpu_has_lsx) + return 0; + + return test_thread_flag(TIF_LSX_CTX_LIVE); +} + +static inline int thread_lasx_context_live(void) +{ + if (!cpu_has_lasx) + return 0; + + return test_thread_flag(TIF_LASX_CTX_LIVE); +} + +#endif /* _ASM_FPU_H */ diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h new file mode 100644 index 0000000000000000000000000000000000000000..070155c3ccf44182a3021815285bc2e980ac1c79 --- /dev/null +++ b/arch/loongarch/include/asm/ftrace.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive for + * more details. + * + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#ifndef _ASM_LOONGARCH_FTRACE_H +#define _ASM_LOONGARCH_FTRACE_H + +#define FTRACE_PLT_IDX 0 +#define FTRACE_REGS_PLT_IDX 1 +#define NR_FTRACE_PLTS 2 + +#ifdef CONFIG_FUNCTION_TRACER +#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ + +#ifndef __ASSEMBLY__ +#ifndef CONFIG_DYNAMIC_FTRACE +extern void _mcount(void); +#define mcount _mcount +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + return addr; +} + +struct dyn_arch_ftrace { +}; + +struct dyn_ftrace; +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); +#define ftrace_init_nop ftrace_init_nop + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +#define ARCH_SUPPORTS_FTRACE_OPS 1 +#endif +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* __ASSEMBLY__ */ +#endif /* CONFIG_FUNCTION_TRACER */ +#endif /* _ASM_LOONGARCH_FTRACE_H */ diff --git a/arch/loongarch/include/asm/futex.h b/arch/loongarch/include/asm/futex.h new file mode 100644 index 0000000000000000000000000000000000000000..ce24a9f13f217befa06f7f75dc4d3af148ab8a1d --- /dev/null +++ b/arch/loongarch/include/asm/futex.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_FUTEX_H +#define _ASM_FUTEX_H + +#include +#include +#include +#include +#include + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +{ \ + __asm__ __volatile__( \ + "1: ll.w %1, %4 # __futex_atomic_op\n" \ + " " insn " \n" \ + "2: sc.w $t0, %2 \n" \ + " beq $t0, $zero, 1b \n" \ + "3: \n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) \ + : "=r" (ret), "=&r" (oldval), \ + "=ZC" (*uaddr) \ + : "0" (0), "ZC" (*uaddr), "Jr" (oparg) \ + : "memory", "t0"); \ +} + +static inline int +arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) +{ + int oldval = 0, ret = 0; + + pagefault_disable(); + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("move $t0, %z5", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("add.w $t0, %1, %z5", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or $t0, %1, %z5", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("and $t0, %1, %z5", ret, oldval, uaddr, ~oparg); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor $t0, %1, %z5", ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + pagefault_enable(); + + if (!ret) + *oval = oldval; + + return ret; +} + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) +{ + int ret = 0; + u32 val = 0; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __asm__ __volatile__( + "# futex_atomic_cmpxchg_inatomic \n" + "1: ll.w %1, %3 \n" + " bne %1, %z4, 3f \n" + " or $t0, %z5, $zero \n" + "2: sc.w $t0, %2 \n" + " beq $zero, $t0, 1b \n" + "3: \n" + __WEAK_LLSC_MB + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) + : "+r" (ret), "=&r" (val), "=ZC" (*uaddr) + : "ZC" (*uaddr), "Jr" (oldval), "Jr" (newval) + : "memory", "t0"); + + *uval = val; + + return ret; +} + +#endif /* _ASM_FUTEX_H */ diff --git a/arch/loongarch/include/asm/gpr-num.h b/arch/loongarch/include/asm/gpr-num.h new file mode 100644 index 0000000000000000000000000000000000000000..e0941af20c7e77b380b15696e6aecf7d629ee18b --- /dev/null +++ b/arch/loongarch/include/asm/gpr-num.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GPR_NUM_H +#define __ASM_GPR_NUM_H + +#ifdef __ASSEMBLY__ + + .equ .L__gpr_num_zero, 0 + .irp num,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .equ .L__gpr_num_$r\num, \num + .endr + +#else /* __ASSEMBLY__ */ + +#define __DEFINE_ASM_GPR_NUMS \ +" .equ .L__gpr_num_zero, 0\n" \ +" .irp num,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ +" .equ .L__gpr_num_$r\\num, \\num\n" \ +" .endr\n" \ + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_GPR_NUM_H */ diff --git a/arch/loongarch/include/asm/hardirq.h b/arch/loongarch/include/asm/hardirq.h new file mode 100644 index 0000000000000000000000000000000000000000..a16d425a03676377f6389aa2414b709cf1b60a05 --- /dev/null +++ b/arch/loongarch/include/asm/hardirq.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_HARDIRQ_H +#define _ASM_HARDIRQ_H + +#include +#include +#include + +extern void ack_bad_irq(unsigned int irq); +#define ack_bad_irq ack_bad_irq + +#define NR_IPI 2 + +typedef struct { + unsigned int ipi_irqs[NR_IPI]; + unsigned int __softirq_pending; +} ____cacheline_aligned irq_cpustat_t; + +#include /* Standard mappings for irq_cpustat_t above */ + +#define __ARCH_IRQ_STAT + +#endif /* _ASM_HARDIRQ_H */ diff --git a/arch/loongarch/include/asm/highmem.h b/arch/loongarch/include/asm/highmem.h new file mode 100644 index 0000000000000000000000000000000000000000..c555be1008d47e9197cc4581d09d2338f7189119 --- /dev/null +++ b/arch/loongarch/include/asm/highmem.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * highmem.h: virtual kernel memory mappings for high memory + * + * Used in CONFIG_HIGHMEM systems for memory pages which + * are not addressable by direct kernel virtual addresses. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_HIGHMEM_H +#define _ASM_HIGHMEM_H + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include + +/* declarations for highmem.c */ +extern unsigned long highstart_pfn, highend_pfn; + +extern pte_t *pkmap_page_table; + +/* + * Right now we initialize only a single pte table. It can be extended + * easily, subsequent pte tables have to be allocated in one physical + * chunk of RAM. + */ +#define LAST_PKMAP 1024 + +#define LAST_PKMAP_MASK (LAST_PKMAP-1) +#define PKMAP_NR(virt) ((virt-PKMAP_BASE) >> PAGE_SHIFT) +#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT)) + +extern void *kmap_high(struct page *page); +extern void kunmap_high(struct page *page); + +extern void *kmap(struct page *page); +extern void kunmap(struct page *page); +extern void *kmap_atomic(struct page *page); +extern void *kmap_atomic_pfn(unsigned long pfn); + +#define flush_cache_kmaps() BUG_ON(cpu_has_dc_aliases) + +extern void kmap_init(void); + +#define kmap_prot PAGE_KERNEL + +#endif /* __KERNEL__ */ + +#endif /* _ASM_HIGHMEM_H */ diff --git a/arch/loongarch/include/asm/hugetlb.h b/arch/loongarch/include/asm/hugetlb.h new file mode 100644 index 0000000000000000000000000000000000000000..0729eb9441ff16bef49a79e542c3c5c133ac34ee --- /dev/null +++ b/arch/loongarch/include/asm/hugetlb.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#ifndef __ASM_HUGETLB_H +#define __ASM_HUGETLB_H + +#include + +uint64_t pmd_to_entrylo(unsigned long pmd_val); + +#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE +static inline int prepare_hugepage_range(struct file *file, + unsigned long addr, + unsigned long len) +{ + unsigned long task_size = STACK_TOP; + struct hstate *h = hstate_file(file); + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if (addr & ~huge_page_mask(h)) + return -EINVAL; + if (len > task_size) + return -ENOMEM; + if (task_size - len < addr) + return -EINVAL; + return 0; +} + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pte_t clear; + pte_t pte = *ptep; + + pte_val(clear) = (unsigned long)invalid_pte_table; + set_pte_at(mm, addr, ptep, clear); + return pte; +} + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); + flush_tlb_page(vma, addr); +} + +#define __HAVE_ARCH_HUGE_PTE_NONE +static inline int huge_pte_none(pte_t pte) +{ + unsigned long val = pte_val(pte) & ~_PAGE_GLOBAL; + return !val || (val == (unsigned long)invalid_pte_table); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep, pte_t pte, + int dirty) +{ + int changed = !pte_same(*ptep, pte); + + if (changed) { + set_pte_at(vma->vm_mm, addr, ptep, pte); + /* + * There could be some standard sized pages in there, + * get them all. + */ + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); + } + return changed; +} + +#include + +#endif /* __ASM_HUGETLB_H */ diff --git a/arch/loongarch/include/asm/hw_irq.h b/arch/loongarch/include/asm/hw_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..1633972591a04f6d679f922812c8d0056fb37e7f --- /dev/null +++ b/arch/loongarch/include/asm/hw_irq.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2000, 2001, 2002 by Ralf Baechle + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_HW_IRQ_H +#define __ASM_HW_IRQ_H + +#include + +extern atomic_t irq_err_count; + +/* + * interrupt-retrigger: NOP for now. This may not be appropriate for all + * machines, we'll see ... + */ + +#endif /* __ASM_HW_IRQ_H */ diff --git a/arch/loongarch/include/asm/idle.h b/arch/loongarch/include/asm/idle.h new file mode 100644 index 0000000000000000000000000000000000000000..f7f2b7dbf958375c462282b8a3e4096d85482db0 --- /dev/null +++ b/arch/loongarch/include/asm/idle.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_IDLE_H +#define __ASM_IDLE_H + +#include + +extern asmlinkage void __arch_cpu_idle(void); + +#endif /* __ASM_IDLE_H */ diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h new file mode 100644 index 0000000000000000000000000000000000000000..380885072375597240db26ac3553e4ee92769210 --- /dev/null +++ b/arch/loongarch/include/asm/inst.h @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Format of an instruction in memory. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_INST_H +#define _ASM_INST_H +#include + +#include +#include +#include + +#include + +/* HACHACHAHCAHC ... */ + +/* In case some other massaging is needed, keep LOONGARCHInst as wrapper */ + +#define LOONGARCHInst(x) x + +#define I_OPCODE_SFT 26 +#define LOONGARCHInst_OPCODE(x) (LOONGARCHInst(x) >> I_OPCODE_SFT) + +#define I_JTARGET_SFT 0 +#define LOONGARCHInst_JTARGET(x) (LOONGARCHInst(x) & 0x03ffffff) + +#define I_RS_SFT 21 +#define LOONGARCHInst_RS(x) ((LOONGARCHInst(x) & 0x03e00000) >> I_RS_SFT) + +#define I_RT_SFT 16 +#define LOONGARCHInst_RT(x) ((LOONGARCHInst(x) & 0x001f0000) >> I_RT_SFT) + +#define I_IMM_SFT 0 +#define LOONGARCHInst_SIMM(x) ((int)((short)(LOONGARCHInst(x) & 0xffff))) +#define LOONGARCHInst_UIMM(x) (LOONGARCHInst(x) & 0xffff) + +#define I_CACHEOP_SFT 18 +#define LOONGARCHInst_CACHEOP(x) ((LOONGARCHInst(x) & 0x001c0000) >> I_CACHEOP_SFT) + +#define I_CACHESEL_SFT 16 +#define LOONGARCHInst_CACHESEL(x) ((LOONGARCHInst(x) & 0x00030000) >> I_CACHESEL_SFT) + +#define I_RD_SFT 11 +#define LOONGARCHInst_RD(x) ((LOONGARCHInst(x) & 0x0000f800) >> I_RD_SFT) + +#define I_RE_SFT 6 +#define LOONGARCHInst_RE(x) ((LOONGARCHInst(x) & 0x000007c0) >> I_RE_SFT) + +#define I_FUNC_SFT 0 +#define LOONGARCHInst_FUNC(x) (LOONGARCHInst(x) & 0x0000003f) + +#define I_FFMT_SFT 21 +#define LOONGARCHInst_FFMT(x) ((LOONGARCHInst(x) & 0x01e00000) >> I_FFMT_SFT) + +#define I_FT_SFT 16 +#define LOONGARCHInst_FT(x) ((LOONGARCHInst(x) & 0x001f0000) >> I_FT_SFT) + +#define I_FS_SFT 11 +#define LOONGARCHInst_FS(x) ((LOONGARCHInst(x) & 0x0000f800) >> I_FS_SFT) + +#define I_FD_SFT 6 +#define LOONGARCHInst_FD(x) ((LOONGARCHInst(x) & 0x000007c0) >> I_FD_SFT) + +#define I_FR_SFT 21 +#define LOONGARCHInst_FR(x) ((LOONGARCHInst(x) & 0x03e00000) >> I_FR_SFT) + +#define I_FMA_FUNC_SFT 2 +#define LOONGARCHInst_FMA_FUNC(x) ((LOONGARCHInst(x) & 0x0000003c) >> I_FMA_FUNC_SFT) + +#define I_FMA_FFMT_SFT 0 +#define LOONGARCHInst_FMA_FFMT(x) (LOONGARCHInst(x) & 0x00000003) + +struct pt_regs; +typedef unsigned int loongarch_instruction; + +/* Recode table from 16-bit register notation to 32-bit GPR. Do NOT export!!! */ +extern const int reg16to32[]; + +#define LOONGARCH_INSN_SIZE sizeof(union loongarch_instruction) + +enum loongarch_gpr { + LOONGARCH_GPR_ZERO = 0, + LOONGARCH_GPR_RA = 1, + LOONGARCH_GPR_TP = 2, + LOONGARCH_GPR_SP = 3, + LOONGARCH_GPR_A0 = 4, /* Reused as V0 for return value */ + LOONGARCH_GPR_A1, /* Reused as V1 for return value */ + LOONGARCH_GPR_A2, + LOONGARCH_GPR_A3, + LOONGARCH_GPR_A4, + LOONGARCH_GPR_A5, + LOONGARCH_GPR_A6, + LOONGARCH_GPR_A7, + LOONGARCH_GPR_T0 = 12, + LOONGARCH_GPR_T1, + LOONGARCH_GPR_T2, + LOONGARCH_GPR_T3, + LOONGARCH_GPR_T4, + LOONGARCH_GPR_T5, + LOONGARCH_GPR_T6, + LOONGARCH_GPR_T7, + LOONGARCH_GPR_T8, + LOONGARCH_GPR_FP = 22, + LOONGARCH_GPR_S0 = 23, + LOONGARCH_GPR_S1, + LOONGARCH_GPR_S2, + LOONGARCH_GPR_S3, + LOONGARCH_GPR_S4, + LOONGARCH_GPR_S5, + LOONGARCH_GPR_S6, + LOONGARCH_GPR_S7, + LOONGARCH_GPR_S8, + LOONGARCH_GPR_MAX +}; + +#define INSN_NOP 0x03400000 +#define INSN_BREAK 0x002a0000 + +#define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000 +#define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000 +#define ADDR_IMMMASK_LU52ID 0xFFF0000000000000 + +#define ADDR_IMMSHIFT_ADDU16ID 16 +#define ADDR_IMMSHIFT_LU32ID 32 +#define ADDR_IMMSHIFT_LU52ID 52 + +#define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN) + +void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc); +unsigned long unaligned_read(void __user *addr, void *value, unsigned long n, bool sign); +unsigned long unaligned_write(void __user *addr, unsigned long value, unsigned long n); + +static inline bool cond_beqz(struct pt_regs *regs, int rj) +{ + return regs->regs[rj] == 0; +} + +static inline bool cond_bnez(struct pt_regs *regs, int rj) +{ + return regs->regs[rj] != 0; +} + +static inline bool cond_beq(struct pt_regs *regs, int rj, int rd) +{ + return regs->regs[rj] == regs->regs[rd]; +} + +static inline bool cond_bne(struct pt_regs *regs, int rj, int rd) +{ + return regs->regs[rj] != regs->regs[rd]; +} + +static inline bool cond_blt(struct pt_regs *regs, int rj, int rd) +{ + return (long)regs->regs[rj] < (long)regs->regs[rd]; +} + +static inline bool cond_bge(struct pt_regs *regs, int rj, int rd) +{ + return (long)regs->regs[rj] >= (long)regs->regs[rd]; +} + +static inline bool cond_bltu(struct pt_regs *regs, int rj, int rd) +{ + return regs->regs[rj] < regs->regs[rd]; +} + +static inline bool cond_bgeu(struct pt_regs *regs, int rj, int rd) +{ + return regs->regs[rj] >= regs->regs[rd]; +} + +static inline bool is_branch_insn(union loongarch_instruction insn) +{ + return insn.reg1i21_format.opcode >= beqz_op && + insn.reg1i21_format.opcode <= bgeu_op; +} + +static inline bool is_pc_insn(union loongarch_instruction insn) +{ + return insn.reg1i20_format.opcode >= pcaddi_op && + insn.reg1i20_format.opcode <= pcaddu18i_op; +} + +unsigned long bs_dest_16(unsigned long now, unsigned int si); +unsigned long bs_dest_21(unsigned long now, unsigned int h, unsigned int l); +unsigned long bs_dest_26(unsigned long now, unsigned int h, unsigned int l); + +int simu_branch(struct pt_regs *regs, union loongarch_instruction insn); +int simu_pc(struct pt_regs *regs, union loongarch_instruction insn); + +int larch_insn_read(void *addr, u32 *insnp); +int larch_insn_write(void *addr, u32 insn); +int larch_insn_patch_text(void *addr, u32 insn); + +u32 larch_insn_gen_nop(void); +u32 larch_insn_gen_b(unsigned long pc, unsigned long dest); +u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest); + +u32 larch_insn_gen_addu16id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); +u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); +u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); + +u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, + unsigned long pc, unsigned long dest); + +u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, + enum loongarch_gpr rk); +u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj); + +#endif /* _ASM_INST_H */ diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h new file mode 100644 index 0000000000000000000000000000000000000000..2fcc2db6f87e46da520e05251eeb8e61dd4fcbda --- /dev/null +++ b/arch/loongarch/include/asm/io.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_IO_H +#define _ASM_IO_H + +#define ARCH_HAS_IOREMAP_WC + +#include +#include + +#include +#include +#include +#include +#include + +/* + * Change "struct page" to physical address. + */ +#define page_to_phys(page) ((phys_addr_t)page_to_pfn(page) << PAGE_SHIFT) + +extern void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size); +extern void __init early_iounmap(void __iomem *addr, unsigned long size); + +#define early_memremap early_ioremap +#define early_memunmap early_iounmap + +#ifdef CONFIG_ARCH_IOREMAP + +static inline void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, + unsigned long prot_val) +{ + if (prot_val & _CACHE_CC) + return (void __iomem *)(unsigned long)(CACHE_BASE + offset); + else + return (void __iomem *)(unsigned long)(UNCACHE_BASE + offset); +} + +#define ioremap(offset, size) \ + ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL_SUC)) + +#define iounmap(addr) ((void)(addr)) + +#endif + +/* + * On LoongArch, ioremap() has two variants, ioremap_wc() and ioremap_cache(). + * They map bus memory into CPU space, the mapped memory is marked uncachable + * (_CACHE_SUC), uncachable but accelerated by write-combine (_CACHE_WUC) and + * cachable (_CACHE_CC) respectively for CPU access. + * + * @offset: bus address of the memory + * @size: size of the resource to map + */ +#define ioremap_wc(offset, size) \ + ioremap_prot((offset), (size), \ + pgprot_val(wc_enabled ? PAGE_KERNEL_WUC : PAGE_KERNEL_SUC)) + +#define ioremap_cache(offset, size) \ + ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL)) + +#define mmiowb() wmb() + +/* + * String version of I/O memory access operations. + */ +extern void __memset_io(volatile void __iomem *dst, int c, size_t count); +extern void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count); +extern void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count); +#define memset_io(c, v, l) __memset_io((c), (v), (l)) +#define memcpy_fromio(a, c, l) __memcpy_fromio((a), (c), (l)) +#define memcpy_toio(c, a, l) __memcpy_toio((c), (a), (l)) + +#include + +#define ARCH_HAS_VALID_PHYS_ADDR_RANGE +extern int valid_phys_addr_range(phys_addr_t addr, size_t size); +extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); + +#endif /* _ASM_IO_H */ diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h new file mode 100644 index 0000000000000000000000000000000000000000..e8c81770bb341424eba3667c9c33c379ea7f9c96 --- /dev/null +++ b/arch/loongarch/include/asm/irq.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_IRQ_H +#define _ASM_IRQ_H + +#include +#include +#include + +#define IRQ_STACK_SIZE THREAD_SIZE +#define IRQ_STACK_START (IRQ_STACK_SIZE - 16) + +DECLARE_PER_CPU(unsigned long, irq_stack); + +/* + * The highest address on the IRQ stack contains a dummy frame put down in + * genex.S (except_vec_vi_handler) which is structured as follows: + * + * top ------------ + * | task sp | <- irq_stack[cpu] + IRQ_STACK_START + * ------------ + * | | <- First frame of IRQ context + * ------------ + * + * task sp holds a copy of the task stack pointer where the struct pt_regs + * from exception entry can be found. + */ + +static inline bool on_irq_stack(int cpu, unsigned long sp) +{ + unsigned long low = per_cpu(irq_stack, cpu); + unsigned long high = low + IRQ_STACK_SIZE; + + return (low <= sp && sp <= high); +} + +struct irq_data; +struct device_node; + +int get_ipi_irq(void); +int get_pmc_irq(void); +int get_timer_irq(void); +void arch_init_irq(void); +void spurious_interrupt(void); +struct irq_domain *loongarch_cpu_irq_init(void); + +#define NR_IRQS_LEGACY 16 + +void arch_trigger_cpumask_backtrace(const struct cpumask *mask, + bool exclude_self); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace + +#endif /* _ASM_IRQ_H */ diff --git a/arch/loongarch/include/asm/irq_regs.h b/arch/loongarch/include/asm/irq_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..92e72213944e68e5c7737bd5805be29d521a0274 --- /dev/null +++ b/arch/loongarch/include/asm/irq_regs.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_IRQ_REGS_H +#define __ASM_IRQ_REGS_H + +#define ARCH_HAS_OWN_IRQ_REGS + +#include + +static inline struct pt_regs *get_irq_regs(void) +{ + return current_thread_info()->regs; +} + +static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs) +{ + struct pt_regs *old_regs; + + old_regs = get_irq_regs(); + current_thread_info()->regs = new_regs; + + return old_regs; +} + +#endif /* __ASM_IRQ_REGS_H */ diff --git a/arch/loongarch/include/asm/irqflags.h b/arch/loongarch/include/asm/irqflags.h new file mode 100644 index 0000000000000000000000000000000000000000..048d5a30ac292bc0c4667f3bac1ccf64a6fc41af --- /dev/null +++ b/arch/loongarch/include/asm/irqflags.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include + +static inline void arch_local_irq_enable(void) +{ + u32 flags = CSR_CRMD_IE; + __asm__ __volatile__( + "csrxchg %[val], %[mask], %[reg]\n\t" + : [val] "+r" (flags) + : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : "memory"); +} + +static inline void arch_local_irq_disable(void) +{ + u32 flags = 0; + __asm__ __volatile__( + "csrxchg %[val], %[mask], %[reg]\n\t" + : [val] "+r" (flags) + : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : "memory"); +} + +static inline unsigned long arch_local_irq_save(void) +{ + u32 flags = 0; + __asm__ __volatile__( + "csrxchg %[val], %[mask], %[reg]\n\t" + : [val] "+r" (flags) + : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : "memory"); + return flags; +} + +static inline void arch_local_irq_restore(unsigned long flags) +{ + __asm__ __volatile__( + "csrxchg %[val], %[mask], %[reg]\n\t" + : [val] "+r" (flags) + : [mask] "r" (CSR_CRMD_IE), [reg] "i" (LOONGARCH_CSR_CRMD) + : "memory"); +} + +static inline unsigned long arch_local_save_flags(void) +{ + u32 flags; + __asm__ __volatile__( + "csrrd %[val], %[reg]\n\t" + : [val] "=r" (flags) + : [reg] "i" (LOONGARCH_CSR_CRMD) + : "memory"); + return flags; +} + +static inline int arch_irqs_disabled_flags(unsigned long flags) +{ + return !(flags & CSR_CRMD_IE); +} + +static inline int arch_irqs_disabled(void) +{ + return arch_irqs_disabled_flags(arch_local_save_flags()); +} + +#endif /* #ifndef __ASSEMBLY__ */ + +#endif /* _ASM_IRQFLAGS_H */ diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h new file mode 100644 index 0000000000000000000000000000000000000000..3cea299a5ef58313a305f7d5a086ae9a32e8aa95 --- /dev/null +++ b/arch/loongarch/include/asm/jump_label.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 Loongson Technology Corporation Limited + * + * Based on arch/arm64/include/asm/jump_label.h + */ +#ifndef __ASM_JUMP_LABEL_H +#define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include + +#define JUMP_LABEL_NOP_SIZE 4 + +#define JUMP_TABLE_ENTRY \ + ".pushsection __jump_table, \"aw\" \n\t" \ + ".align 3 \n\t" \ + ".long 1b - ., %l[l_yes] - . \n\t" \ + ".quad %0 - . \n\t" \ + ".popsection \n\t" + +static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) +{ + asm_volatile_goto( + "1: nop \n\t" + JUMP_TABLE_ENTRY + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; + +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) +{ + asm_volatile_goto( + "1: b %l[l_yes] \n\t" + JUMP_TABLE_ENTRY + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; + +l_yes: + return true; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/loongarch/include/asm/kasan.h b/arch/loongarch/include/asm/kasan.h new file mode 100644 index 0000000000000000000000000000000000000000..e0d430ce9775a8f56326159212ff5d7456acd2bf --- /dev/null +++ b/arch/loongarch/include/asm/kasan.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include +#include + +#define KASAN_SHADOW_SCALE_SHIFT 3 +#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) + +#define XRANGE_SHIFT (48) + +/* Valid address length */ +#define XRANGE_SHADOW_SHIFT (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3) +/* Used for taking out the valid address */ +#define XRANGE_SHADOW_MASK GENMASK_ULL(XRANGE_SHADOW_SHIFT - 1, 0) +/* One segment whole address space size */ +#define XRANGE_SIZE (XRANGE_SHADOW_MASK + 1) + +/* 64-bit segment value. */ +#define XKPRANGE_UC_SEG (0x8000) +#define XKPRANGE_CC_SEG (0x9000) +#define XKVRANGE_VC_SEG (0xffff) + +/* Cached */ +#define XKPRANGE_CC_START CACHE_BASE +#define XKPRANGE_CC_SIZE XRANGE_SIZE +#define XKPRANGE_CC_KASAN_OFFSET (0) +#define XKPRANGE_CC_SHADOW_SIZE (XKPRANGE_CC_SIZE >> KASAN_SHADOW_SCALE_SHIFT) +#define XKPRANGE_CC_SHADOW_END (XKPRANGE_CC_KASAN_OFFSET + XKPRANGE_CC_SHADOW_SIZE) + +/* UnCached */ +#define XKPRANGE_UC_START UNCACHE_BASE +#define XKPRANGE_UC_SIZE XRANGE_SIZE +#define XKPRANGE_UC_KASAN_OFFSET XKPRANGE_CC_SHADOW_END +#define XKPRANGE_UC_SHADOW_SIZE (XKPRANGE_UC_SIZE >> KASAN_SHADOW_SCALE_SHIFT) +#define XKPRANGE_UC_SHADOW_END (XKPRANGE_UC_KASAN_OFFSET + XKPRANGE_UC_SHADOW_SIZE) + +/* VMALLOC (Cached or UnCached) */ +#define XKVRANGE_VC_START VMALLOC_START +#define XKVRANGE_VC_SIZE round_up(VMEMMAP_END - MODULES_VADDR + 1, PGDIR_SIZE) +#define XKVRANGE_VC_KASAN_OFFSET XKPRANGE_UC_SHADOW_END +#define XKVRANGE_VC_SHADOW_SIZE (XKVRANGE_VC_SIZE >> KASAN_SHADOW_SCALE_SHIFT) +#define XKVRANGE_VC_SHADOW_END (XKVRANGE_VC_KASAN_OFFSET + XKVRANGE_VC_SHADOW_SIZE) + +/* Kasan shadow memory start right after vmalloc. */ +#define KASAN_SHADOW_START round_up(VMEMMAP_END, PGDIR_SIZE) +#define KASAN_SHADOW_SIZE (XKVRANGE_VC_SHADOW_END - XKPRANGE_CC_KASAN_OFFSET) +#define KASAN_SHADOW_END round_up(KASAN_SHADOW_START + KASAN_SHADOW_SIZE, PGDIR_SIZE) + +#define XKPRANGE_CC_SHADOW_OFFSET (KASAN_SHADOW_START + XKPRANGE_CC_KASAN_OFFSET) +#define XKPRANGE_UC_SHADOW_OFFSET (KASAN_SHADOW_START + XKPRANGE_UC_KASAN_OFFSET) +#define XKVRANGE_VC_SHADOW_OFFSET (KASAN_SHADOW_START + XKVRANGE_VC_KASAN_OFFSET) + +extern bool kasan_early_stage; +extern unsigned char kasan_early_shadow_page[PAGE_SIZE]; + +#define kasan_mem_to_shadow kasan_mem_to_shadow +void *kasan_mem_to_shadow(const void *addr); + +#define kasan_shadow_to_mem kasan_shadow_to_mem +const void *kasan_shadow_to_mem(const void *shadow_addr); + +#define addr_has_metadata addr_has_metadata +static inline bool addr_has_metadata(const void *addr) +{ + return (kasan_mem_to_shadow((void *)addr) != NULL); +} + +void kasan_init(void); +asmlinkage void kasan_early_init(void); + +#endif +#endif diff --git a/arch/loongarch/include/asm/kdebug.h b/arch/loongarch/include/asm/kdebug.h new file mode 100644 index 0000000000000000000000000000000000000000..64cdaead21a8a0ebce48fdcd00623d2d02a4186d --- /dev/null +++ b/arch/loongarch/include/asm/kdebug.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_LOONGARCH_KDEBUG_H +#define _ASM_LOONGARCH_KDEBUG_H + +#include + +enum die_val { + DIE_OOPS = 1, + DIE_RI, + DIE_FP, + DIE_SIMD, + DIE_TRAP, + DIE_PAGE_FAULT, + DIE_BREAK, + DIE_SSTEPBP, + DIE_UPROBE, + DIE_UPROBE_XOL, +}; + +#endif /* _ASM_LOONGARCH_KDEBUG_H */ diff --git a/arch/loongarch/include/asm/kexec.h b/arch/loongarch/include/asm/kexec.h new file mode 100644 index 0000000000000000000000000000000000000000..5a32604c804784c0b01fd20da192480c3b865bfe --- /dev/null +++ b/arch/loongarch/include/asm/kexec.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * kexec.h for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_KEXEC_H +#define _ASM_KEXEC_H + +#include +#include + +/* Maximum physical address we can use pages from */ +#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) +/* Maximum address we can reach in physical address mode */ +#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL) + /* Maximum address we can use for the control code buffer */ +#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL) + +/* Reserve a page for the control code buffer */ +#define KEXEC_CONTROL_PAGE_SIZE PAGE_SIZE + +/* The native architecture */ +#define KEXEC_ARCH KEXEC_ARCH_LOONGARCH + +static inline void crash_setup_regs(struct pt_regs *newregs, + struct pt_regs *oldregs) +{ + if (oldregs) + memcpy(newregs, oldregs, sizeof(*newregs)); + else + prepare_frametrace(newregs); +} + +#define ARCH_HAS_KIMAGE_ARCH + +struct kimage_arch { + unsigned long efi_boot; + unsigned long cmdline_ptr; + unsigned long systable_ptr; +}; + +typedef void (*do_kexec_t)(unsigned long efi_boot, + unsigned long cmdline_ptr, + unsigned long systable_ptr, + unsigned long start_addr, + unsigned long first_ind_entry, + unsigned long kdump_reloc_offset); + +struct kimage; +extern const unsigned char relocate_new_kernel[]; +extern const size_t relocate_new_kernel_size; +extern void kexec_reboot(void); + +#ifdef CONFIG_SMP +extern atomic_t kexec_ready_to_reboot; +extern const unsigned char kexec_smp_wait[]; +#endif + +#endif /* !_ASM_KEXEC_H */ diff --git a/arch/loongarch/include/asm/kgdb.h b/arch/loongarch/include/asm/kgdb.h new file mode 100644 index 0000000000000000000000000000000000000000..91b1e30db000931a674f53d186756c6dd69b184a --- /dev/null +++ b/arch/loongarch/include/asm/kgdb.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_KGDB_H_ +#define __ASM_KGDB_H_ + +#ifdef __KERNEL__ + +#ifdef CONFIG_32BIT +#define KGDB_GDB_REG_SIZE 32 +#define GDB_SIZEOF_REG sizeof(u32) +#else /* CONFIG_CPU_32BIT */ +#define KGDB_GDB_REG_SIZE 64 +#define GDB_SIZEOF_REG sizeof(u64) +#endif + +#define BUFMAX 2048 +#define DBG_ALL_REG_NUM 76 +#define DBG_MAX_REG_NUM 35 +#define NUMREGBYTES (DBG_MAX_REG_NUM * sizeof(GDB_SIZEOF_REG)) +#define NUMCRITREGBYTES (12 * sizeof(GDB_SIZEOF_REG)) +#define BREAK_INSTR_SIZE 4 +#define CACHE_FLUSH_IS_SAFE 0 + +extern void arch_kgdb_breakpoint(void); +extern void *saved_vectors[32]; +extern void handle_exception(struct pt_regs *regs); +extern void breakinst(void); +extern int kgdb_ll_trap(int cmd, const char *str, + struct pt_regs *regs, long err, int trap, int sig); + +#endif /* __KERNEL__ */ + +#endif /* __ASM_KGDB_H_ */ diff --git a/arch/loongarch/include/asm/kmap_types.h b/arch/loongarch/include/asm/kmap_types.h new file mode 100644 index 0000000000000000000000000000000000000000..16665dc2431b67a4a14c8cdc78340029b28dcdfb --- /dev/null +++ b/arch/loongarch/include/asm/kmap_types.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_KMAP_TYPES_H +#define _ASM_KMAP_TYPES_H + +#ifdef CONFIG_DEBUG_HIGHMEM +#define __WITH_KM_FENCE +#endif + +#include + +#undef __WITH_KM_FENCE + +#endif diff --git a/arch/loongarch/include/asm/kprobes.h b/arch/loongarch/include/asm/kprobes.h new file mode 100644 index 0000000000000000000000000000000000000000..bac015197c7a061757933a453b719c23d8474332 --- /dev/null +++ b/arch/loongarch/include/asm/kprobes.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel Probes (KProbes) + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _ASM_KPROBES_H +#define _ASM_KPROBES_H + +#include + +#ifdef CONFIG_KPROBES +#include +#include + +#include +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT + +struct kprobe; +struct pt_regs; + +typedef union loongarch_instruction kprobe_opcode_t; + +#define MAX_INSN_SIZE 2 + +#define flush_insn_slot(p) \ +do { \ + if (p->addr) \ + flush_icache_range((unsigned long)p->addr, \ + (unsigned long)p->addr + \ + (MAX_INSN_SIZE * sizeof(kprobe_opcode_t))); \ +} while (0) + + +#define kretprobe_blacklist_size 0 + +void arch_remove_kprobe(struct kprobe *p); +int kprobe_fault_handler(struct pt_regs *regs, int trapnr); + +/* Architecture specific copy of original instruction*/ +struct arch_specific_insn { + /* copy of the original instruction */ + kprobe_opcode_t *insn; +}; + +struct prev_kprobe { + struct kprobe *kp; + unsigned long status; + unsigned long old_SR; + unsigned long saved_SR; + unsigned long saved_era; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned long kprobe_status; + unsigned long kprobe_old_SR; + unsigned long kprobe_saved_SR; + unsigned long kprobe_saved_era; + /* Per-thread fields, used while emulating branches */ + unsigned long flags; + unsigned long target_era; + struct prev_kprobe prev_kprobe; +}; + +extern int kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data); + +#endif /* CONFIG_KPROBES */ +#endif /* _ASM_KPROBES_H */ diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h new file mode 100644 index 0000000000000000000000000000000000000000..fcc5c1e2b2ad33497b23798290466bce356710fb --- /dev/null +++ b/arch/loongarch/include/asm/kvm_host.h @@ -0,0 +1,356 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGARCH_KVM_HOST_H__ +#define __LOONGARCH_KVM_HOST_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define __KVM_HAVE_ARCH_VCPU_DEBUGFS + +/* Loongarch KVM register ids */ +#define LOONGARCH_CSR_32(_R, _S) \ + (KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U32 | (8 * (_R) + (_S))) + +#define LOONGARCH_CSR_64(_R, _S) \ + (KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | (8 * (_R) + (_S))) + +#define KVM_IOC_CSRID(id) LOONGARCH_CSR_64(id, 0) +#define KVM_GET_IOC_CSRIDX(id) ((id & KVM_CSR_IDX_MASK) >> 3) + +#define LOONGSON_VIRT_REG_BASE 0x1f000000 +#define KVM_MAX_VCPUS 256 +#define KVM_USER_MEM_SLOTS 256 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 0 + +#define KVM_HALT_POLL_NS_DEFAULT 500000 + +#define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(1) +#define KVM_REQ_EVENT KVM_ARCH_REQ(4) + +#define KVM_INVALID_ADDR 0xdeadbeef +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +struct kvm_vm_stat { + ulong remote_tlb_flush; + u64 vm_ioctl_irq_line; + u64 ls7a_ioapic_update; + u64 ls7a_ioapic_set_irq; + u64 ioapic_reg_write; + u64 ioapic_reg_read; + u64 set_ls7a_ioapic; + u64 get_ls7a_ioapic; + u64 set_ls3a_ext_irq; + u64 get_ls3a_ext_irq; + u64 trigger_ls3a_ext_irq; + u64 pip_read_exits; + u64 pip_write_exits; + u64 ls7a_msi_irq; +}; +struct kvm_vcpu_stat { + u64 excep_exits[EXCCODE_INT_START]; + u64 idle_exits; + u64 signal_exits; + u64 int_exits; + u64 rdcsr_cpu_feature_exits; + u64 rdcsr_misc_func_exits; + u64 rdcsr_ipi_access_exits; + u64 cpucfg_exits; + u64 huge_dec_exits; + u64 huge_thp_exits; + u64 huge_adjust_exits; + u64 huge_set_exits; + u64 huge_merge_exits; + u64 halt_successful_poll; + u64 halt_attempted_poll; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; + u64 halt_poll_invalid; + u64 halt_wakeup; +}; + +#define KVM_MEMSLOT_DISABLE_THP (1UL << 17) +struct kvm_arch_memory_slot { + unsigned int flags; +}; + +enum { + IOCSR_FEATURES, + IOCSR_VENDOR, + IOCSR_CPUNAME, + IOCSR_NODECNT, + IOCSR_MISC_FUNC, + IOCSR_MAX +}; + +struct kvm_context { + unsigned long gid_mask; + unsigned long gid_ver_mask; + unsigned long gid_fisrt_ver; + unsigned long vpid_cache; + struct kvm_vcpu *last_vcpu; +}; + +struct kvm_arch { + /* Guest physical mm */ + struct mm_struct gpa_mm; + /* Mask of CPUs needing GPA ASID flush */ + cpumask_t asid_flush_mask; + + unsigned char online_vcpus; + unsigned char is_migrate; + s64 stablecounter_gftoffset; + u32 cpucfg_lasx; + struct ls7a_kvm_ioapic *v_ioapic; + struct ls3a_kvm_ipi *v_gipi; + struct ls3a_kvm_routerirq *v_routerirq; + struct ls3a_kvm_extirq *v_extirq; + spinlock_t iocsr_lock; + struct kvm_iocsr_entry iocsr[IOCSR_MAX]; + struct kvm_cpucfg cpucfgs; + struct kvm_context __percpu *vmcs; +}; + + +#define LOONGARCH_CSRS 0x100 +#define CSR_UCWIN_BASE 0x100 +#define CSR_UCWIN_SIZE 0x10 +#define CSR_DMWIN_BASE 0x180 +#define CSR_DMWIN_SIZE 0x4 +#define CSR_PERF_BASE 0x200 +#define CSR_PERF_SIZE 0x8 +#define CSR_DEBUG_BASE 0x500 +#define CSR_DEBUG_SIZE 0x3 +#define CSR_ALL_SIZE 0x800 + +struct loongarch_csrs { + unsigned long csrs[CSR_ALL_SIZE]; +}; + +/* Resume Flags */ +#define RESUME_FLAG_DR (1<<0) /* Reload guest nonvolatile state? */ +#define RESUME_FLAG_HOST (1<<1) /* Resume host? */ + +#define RESUME_GUEST 0 +#define RESUME_GUEST_DR RESUME_FLAG_DR +#define RESUME_HOST RESUME_FLAG_HOST + +enum emulation_result { + EMULATE_DONE, /* no further processing */ + EMULATE_DO_MMIO, /* kvm_run filled with MMIO request */ + EMULATE_FAIL, /* can't emulate this instruction */ + EMULATE_WAIT, /* WAIT instruction */ + EMULATE_PRIV_FAIL, + EMULATE_EXCEPT, /* A guest exception has been generated */ + EMULATE_PV_HYPERCALL, /* HYPCALL instruction */ + EMULATE_DEBUG, /* Emulate guest kernel debug */ + EMULATE_DO_IOCSR, /* handle IOCSR request */ +}; + +#define KVM_LARCH_FPU (0x1 << 0) +#define KVM_LARCH_LSX (0x1 << 1) +#define KVM_LARCH_LASX (0x1 << 2) +#define KVM_LARCH_FP_ALL (KVM_LARCH_FPU | KVM_LARCH_LSX | KVM_LARCH_LASX) +#define KVM_LARCH_DATA_HWBP (0x1 << 3) +#define KVM_LARCH_INST_HWBP (0x1 << 4) +#define KVM_LARCH_HWBP (KVM_LARCH_DATA_HWBP | KVM_LARCH_INST_HWBP) +#define KVM_LARCH_RESET (0x1 << 5) +#define KVM_LARCH_PERF (0x1 << 6) +#define KVM_LARCH_LBT (0x1 << 7) + +struct kvm_vcpu_arch { + unsigned long guest_eentry; + unsigned long host_eentry; + int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu); + int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); + + /* Host registers preserved across guest mode execution */ + unsigned long host_stack; + unsigned long host_gp; + unsigned long host_pgd; + unsigned long host_pgdhi; + unsigned long host_entryhi; + + /* Host CSR registers used when handling exits from guest */ + unsigned long badv; + unsigned long host_estat; + unsigned long badi; + unsigned long host_ecfg; + unsigned long host_percpu; + + u32 is_hypcall; + /* GPRS */ + unsigned long gprs[32]; + unsigned long pc; + + /* FPU State */ + struct loongarch_fpu fpu FPU_ALIGN; + struct loongarch_lbt lbt; + /* Which auxiliary state is loaded (KVM_LOONGARCH_AUX_*) */ + unsigned int aux_inuse; + + /* CSR State */ + struct loongarch_csrs *csr; + + /* GPR used as IO source/target */ + u32 io_gpr; + + struct hrtimer swtimer; + /* Count timer control KVM register */ + u32 count_ctl; + + /* Bitmask of exceptions that are pending */ + unsigned long irq_pending; + /* Bitmask of pending exceptions to be cleared */ + unsigned long irq_clear; + + /* Cache some mmu pages needed inside spinlock regions */ + struct kvm_mmu_memory_cache mmu_page_cache; + + /* vcpu's vpid is different on each host cpu in an smp system */ + u64 vpid[NR_CPUS]; + + /* Period of stable timer tick in ns */ + u64 timer_period; + /* Frequency of stable timer in Hz */ + u64 timer_mhz; + /* Stable bias from the raw time */ + u64 timer_bias; + /* Dynamic nanosecond bias (multiple of timer_period) to avoid overflow */ + s64 timer_dyn_bias; + /* Save ktime */ + ktime_t stable_ktime_saved; + + u64 core_ext_ioisr[4]; + + /* Last CPU the VCPU state was loaded on */ + int last_sched_cpu; + /* Last CPU the VCPU actually executed guest code on */ + int last_exec_cpu; + + u8 fpu_enabled; + u8 lsx_enabled; + /* paravirt steal time */ + struct { + u64 guest_addr; + u64 last_steal; + struct gfn_to_pfn_cache cache; + } st; + + /* pv related host specific info */ + struct { + bool pv_unhalted; + } pv; + + struct kvm_guest_debug_arch guest_debug; + /* save host pmu csr */ + u64 perf_ctrl[4]; + u64 perf_cntr[4]; + +}; + +static inline unsigned long readl_sw_gcsr(struct loongarch_csrs *csr, int reg) +{ + return csr->csrs[reg]; +} + +static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, \ + unsigned long val) +{ + csr->csrs[reg] = val; +} + +/* Helpers */ +static inline bool _kvm_guest_has_fpu(struct kvm_vcpu_arch *arch) +{ + return cpu_has_fpu && arch->fpu_enabled; +} + + +static inline bool _kvm_guest_has_lsx(struct kvm_vcpu_arch *arch) +{ + return cpu_has_lsx && arch->lsx_enabled; +} + +bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu); +void _kvm_init_fault(void); + +/* Debug: dump vcpu state */ +int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); + +/* MMU handling */ +int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write); +void kvm_flush_tlb_all(void); +void _kvm_destroy_mm(struct kvm *kvm); +pgd_t *kvm_pgd_alloc(void); +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); + +enum _kvm_fault_result { + KVM_LOONGARCH_MAPPED = 0, + KVM_LOONGARCH_GVA, + KVM_LOONGARCH_GPA, + KVM_LOONGARCH_TLB, + KVM_LOONGARCH_TLBINV, + KVM_LOONGARCH_TLBMOD, +}; + +#define KVM_ARCH_WANT_MMU_NOTIFIER +int kvm_unmap_hva_range(struct kvm *kvm, + unsigned long start, unsigned long end, bool blockable); +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); + +static inline void update_pc(struct kvm_vcpu_arch *arch) +{ + arch->pc += 4; +} + +/** + * kvm_is_ifetch_fault() - Find whether a TLBL exception is due to ifetch fault. + * @vcpu: Virtual CPU. + * + * Returns: Whether the TLBL exception was likely due to an instruction + * fetch fault rather than a data load fault. + */ +static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *arch) +{ + if (arch->pc == arch->badv) + return true; + + return false; +} + +/* Misc */ +static inline void kvm_arch_hardware_unsetup(void) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} + +extern int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu); +extern void kvm_exception_entry(void); +#endif /* __LOONGARCH_KVM_HOST_H__ */ diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..a14e15b469dee28d0f22bc0264d9e21e118f3913 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_para.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_KVM_PARA_H +#define _ASM_LOONGARCH_KVM_PARA_H + +#include + +#define KVM_HYPERCALL ".word 0x002b8000" +/* + * Hypcall code field + */ +#define KVM_HC_CODE_SERIVCE 0x0 +#define KVM_HC_CODE_SWDBG 0x5 +/* + * function id + * 0x00000 ~ 0xfffff Standard Hypervisor Calls + */ +#define KVM_HC_FUNC_FEATURE 0x0 +#define KVM_HC_FUNC_NOTIFY 0x1 +#define KVM_HC_FUNC_IPI 0x2 +/* + * LoongArch support PV feature list + */ +#define KVM_FEATURE_STEAL_TIME 0 +#define KVM_FEATURE_MULTI_IPI 1 +#define KVM_FEATURE_PARAVIRT_SPINLOCK 2 +/* + * LoongArch hypcall return code + */ +#define KVM_RET_SUC 1 +#define KVM_RET_NOT_SUPPORTED -1 + +#define KVM_VCPU_PREEMPTED (1 << 0) + +/* + * Hypercalls interface for KVM. + * + * a0: function identifier + * a1-a6: args + * Return value will be placed in v0. + * Up to 6 arguments are passed in a1, a2, a3, a4, a5, a6. + */ +static inline long kvm_hypercall0(u64 fid) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall1(u64 fid, unsigned long arg0) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall2(u64 fid, + unsigned long arg0, unsigned long arg1) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1), "r" (a2) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall3(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1), "r" (a2), "r" (a3) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall4(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall5(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + register unsigned long a5 asm("a5") = arg4; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall6(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4, unsigned long arg5) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + register unsigned long a5 asm("a5") = arg4; + register unsigned long a6 asm("a6") = arg5; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6) + : "memory" + ); + + return ret; +} + +static inline bool kvm_check_and_clear_guest_paused(void) +{ + return false; +} + +static inline unsigned int kvm_arch_para_features(void) +{ + return 0; +} + +static inline unsigned int kvm_arch_para_hints(void) +{ + return 0; +} + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +void __init kvm_spinlock_init(void); +#else /* !CONFIG_PARAVIRT_SPINLOCKS */ +static inline void kvm_spinlock_init(void) +{ +} +#endif /* CONFIG_PARAVIRT_SPINLOCKS */ + +#endif /* _ASM_LOONGARCH_KVM_PARA_H */ diff --git a/arch/loongarch/include/asm/kvm_types.h b/arch/loongarch/include/asm/kvm_types.h new file mode 100644 index 0000000000000000000000000000000000000000..9cdc260c35791db52ae2f49177a7b7637fd416ad --- /dev/null +++ b/arch/loongarch/include/asm/kvm_types.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_KVM_TYPES_H +#define _ASM_LOONGARCH_KVM_TYPES_H + +#define KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 4 + +#endif /* _ASM_LOONGARCH_KVM_TYPES_H */ diff --git a/arch/loongarch/include/asm/lbt.h b/arch/loongarch/include/asm/lbt.h new file mode 100644 index 0000000000000000000000000000000000000000..86e4ec9c745b41eb01094f88b4b7da8cf2191af0 --- /dev/null +++ b/arch/loongarch/include/asm/lbt.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + */ +#ifndef _ASM_LBT_H +#define _ASM_LBT_H + +#include +#include +#include + +#ifdef CONFIG_CPU_HAS_LBT + +#define STR(x) __STR(x) +#define __STR(x) #x + +extern void _init_lbt(void); + +static inline void save_lbt_registers(struct loongarch_lbt *prev) +{ + unsigned long tmp = 0; + + __asm__ __volatile__ ( + "movscr2gr %[tmp], $scr0 \n" + "stptr.d %[tmp], %[prev], " STR(THREAD_SCR0) " \n" + "movscr2gr %[tmp], $scr1 \n" + "stptr.d %[tmp], %[prev], " STR(THREAD_SCR1) " \n" + "movscr2gr %[tmp], $scr2 \n" + "stptr.d %[tmp], %[prev], " STR(THREAD_SCR2) " \n" + "movscr2gr %[tmp], $scr3 \n" + "stptr.d %[tmp], %[prev], " STR(THREAD_SCR3) " \n" + "x86mfflag %[tmp], 0x3f \n" + "stptr.d %[tmp], %[prev], " STR(THREAD_EFLAGS) " \n" + : + : [prev] "r" (prev), [tmp] "r" (tmp) + : "memory" + ); +} + +static inline void restore_lbt_registers(struct loongarch_lbt *next) +{ + unsigned long tmp = 0; + + __asm__ __volatile__ ( + "ldptr.d %[tmp], %[next], " STR(THREAD_SCR0) " \n" + "movgr2scr $scr0, %[tmp] \n" + "ldptr.d %[tmp], %[next], " STR(THREAD_SCR1) " \n" + "movgr2scr $scr1, %[tmp] \n" + "ldptr.d %[tmp], %[next], " STR(THREAD_SCR2) " \n" + "movgr2scr $scr2, %[tmp] \n" + "ldptr.d %[tmp], %[next], " STR(THREAD_SCR3) " \n" + "movgr2scr $scr3, %[tmp] \n" + "ldptr.d %[tmp], %[next], " STR(THREAD_EFLAGS) " \n" + "x86mtflag %[tmp], 0x3f \n" + : + : [next] "r" (next), [tmp] "r" (tmp) + : + ); +} + +static inline void enable_lbt(void) +{ + if (cpu_has_lbt) + csr_xchg32(CSR_EUEN_LBTEN, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN); +} + +static inline void disable_lbt(void) +{ + if (cpu_has_lbt) + csr_xchg32(0, CSR_EUEN_LBTEN, LOONGARCH_CSR_EUEN); +} + +static inline int is_lbt_enabled(void) +{ + if (!cpu_has_lbt) + return 0; + + return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LBTEN) ? + 1 : 0; +} + +static inline int is_lbt_owner(void) +{ + return test_thread_flag(TIF_USEDLBT); +} + +static inline void __own_lbt(void) +{ + enable_lbt(); + set_thread_flag(TIF_USEDLBT); + KSTK_EUEN(current) |= CSR_EUEN_LBTEN; +} + +static inline void init_lbt(void) +{ + __own_lbt(); + _init_lbt(); +} + +static inline void own_lbt_inatomic(int restore) +{ + if (cpu_has_lbt && !is_lbt_owner()) { + __own_lbt(); + if (restore) + restore_lbt_registers(¤t->thread.lbt); + } +} + +static inline void lose_lbt_inatomic(int save, struct task_struct *tsk) +{ + if (is_lbt_owner()) { + if (save) + save_lbt_registers(&tsk->thread.lbt); + + disable_lbt(); + clear_tsk_thread_flag(tsk, TIF_USEDLBT); + } + KSTK_EUEN(tsk) &= ~(CSR_EUEN_LBTEN); +} + +static inline void lose_lbt(int save) +{ + preempt_disable(); + lose_lbt_inatomic(save, current); + preempt_enable(); +} + +#else +static inline void own_lbt_inatomic(int restore) {} +static inline void lose_lbt_inatomic(int save, struct task_struct *tsk) {} +static inline void init_lbt(void) {} +static inline void lose_lbt(int save) {} +#endif + +static inline int thread_lbt_context_live(void) +{ + if (__builtin_constant_p(cpu_has_lbt) && !cpu_has_lbt) + return 0; + + return test_thread_flag(TIF_LBT_CTX_LIVE); +} + +#endif diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h new file mode 100644 index 0000000000000000000000000000000000000000..e2eca1a25b4ef7af3f5fb788d2a9e2d8fd6592d8 --- /dev/null +++ b/arch/loongarch/include/asm/linkage.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LINKAGE_H +#define __ASM_LINKAGE_H + +#define __ALIGN .align 2 +#define __ALIGN_STR __stringify(__ALIGN) + +#define SYM_FUNC_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + .cfi_startproc; + +#define SYM_FUNC_START_NOALIGN(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ + .cfi_startproc; + +#define SYM_FUNC_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \ + .cfi_startproc; + +#define SYM_FUNC_START_LOCAL_NOALIGN(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ + .cfi_startproc; + +#define SYM_FUNC_START_WEAK(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \ + .cfi_startproc; + +#define SYM_FUNC_START_WEAK_NOALIGN(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ + .cfi_startproc; + +#define SYM_FUNC_END(name) \ + .cfi_endproc; \ + SYM_END(name, SYM_T_FUNC) + +#define SYM_CODE_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + .cfi_startproc; + +#define SYM_CODE_END(name) \ + .cfi_endproc; \ + SYM_END(name, SYM_T_NONE) + +#endif diff --git a/arch/loongarch/include/asm/livepatch.h b/arch/loongarch/include/asm/livepatch.h new file mode 100644 index 0000000000000000000000000000000000000000..c75cc6c9e25a8c357e654fd1c7ff1bb50dbec22a --- /dev/null +++ b/arch/loongarch/include/asm/livepatch.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LIVEPATCH_H +#define _ASM_LIVEPATCH_H + +#include + +static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) +{ + regs->csr_era = ip; +} +#endif /* _ASM_LIVEPATCH_H */ diff --git a/arch/loongarch/include/asm/local.h b/arch/loongarch/include/asm/local.h new file mode 100644 index 0000000000000000000000000000000000000000..769c809ea77109519eac3662e3a0e00b26339e38 --- /dev/null +++ b/arch/loongarch/include/asm/local.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ARCH_LOONGARCH_LOCAL_H +#define _ARCH_LOONGARCH_LOCAL_H + +#include +#include +#include +#include + +typedef struct +{ + atomic_long_t a; +} local_t; + +#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) } + +#define local_read(l) atomic_long_read(&(l)->a) +#define local_set(l, i) atomic_long_set(&(l)->a, (i)) + +#define local_add(i, l) atomic_long_add((i), (&(l)->a)) +#define local_sub(i, l) atomic_long_sub((i), (&(l)->a)) +#define local_inc(l) atomic_long_inc(&(l)->a) +#define local_dec(l) atomic_long_dec(&(l)->a) + +/* + * Same as above, but return the result value + */ +static __inline__ long local_add_return(long i, local_t * l) +{ + unsigned long result; + + __asm__ __volatile__( + " " __AMADD " %1, %2, %0 \n" + : "+ZB" (l->a.counter), "=&r" (result) + : "r" (i) + : "memory"); + result = result + i; + + return result; +} + +static __inline__ long local_sub_return(long i, local_t * l) +{ + unsigned long result; + + __asm__ __volatile__( + " " __AMADD "%1, %2, %0 \n" + : "+ZB" (l->a.counter), "=&r" (result) + : "r" (-i) + : "memory"); + + result = result - i; + + return result; +} + +#define local_cmpxchg(l, o, n) \ + ((long)cmpxchg_local(&((l)->a.counter), (o), (n))) +#define local_xchg(l, n) (atomic_long_xchg((&(l)->a), (n))) + +/** + * local_add_unless - add unless the number is a given value + * @l: pointer of type local_t + * @a: the amount to add to l... + * @u: ...unless l is equal to u. + * + * Atomically adds @a to @l, so long as it was not @u. + * Returns non-zero if @l was not @u, and zero otherwise. + */ +#define local_add_unless(l, a, u) \ +({ \ + long c, old; \ + c = local_read(l); \ + while (c != (u) && (old = local_cmpxchg((l), c, c + (a))) != c) \ + c = old; \ + c != (u); \ +}) +#define local_inc_not_zero(l) local_add_unless((l), 1, 0) + +#define local_dec_return(l) local_sub_return(1, (l)) +#define local_inc_return(l) local_add_return(1, (l)) + +/* + * local_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @l: pointer of type local_t + * + * Atomically subtracts @i from @l and returns + * true if the result is zero, or false for all + * other cases. + */ +#define local_sub_and_test(i, l) (local_sub_return((i), (l)) == 0) + +/* + * local_inc_and_test - increment and test + * @l: pointer of type local_t + * + * Atomically increments @l by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +#define local_inc_and_test(l) (local_inc_return(l) == 0) + +/* + * local_dec_and_test - decrement by 1 and test + * @l: pointer of type local_t + * + * Atomically decrements @l by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +#define local_dec_and_test(l) (local_sub_return(1, (l)) == 0) + +/* + * local_add_negative - add and test if negative + * @l: pointer of type local_t + * @i: integer value to add + * + * Atomically adds @i to @l and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +#define local_add_negative(i, l) (local_add_return(i, (l)) < 0) + +/* Use these for per-cpu local_t variables: on some archs they are + * much more efficient than these naive implementations. Note they take + * a variable, not an address. + */ + +#define __local_inc(l) ((l)->a.counter++) +#define __local_dec(l) ((l)->a.counter++) +#define __local_add(i, l) ((l)->a.counter+=(i)) +#define __local_sub(i, l) ((l)->a.counter-=(i)) + +#endif /* _ARCH_LOONGARCH_LOCAL_H */ diff --git a/arch/loongarch/include/asm/loongarchregs.h b/arch/loongarch/include/asm/loongarchregs.h new file mode 100644 index 0000000000000000000000000000000000000000..dc6a13d560967f9671ee5ccc3f518c4c59bf4c65 --- /dev/null +++ b/arch/loongarch/include/asm/loongarchregs.h @@ -0,0 +1,1650 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_LOONGARCHREGS_H +#define _ASM_LOONGARCHREGS_H + +#include +#include +#include + +#ifndef __ASSEMBLY__ +#include + +/* + * parse_r var, r - Helper assembler macro for parsing register names. + * + * This converts the register name in $n form provided in \r to the + * corresponding register number, which is assigned to the variable \var. It is + * needed to allow explicit encoding of instructions in inline assembly where + * registers are chosen by the compiler in $n form, allowing us to avoid using + * fixed register numbers. + * + * It also allows newer instructions (not implemented by the assembler) to be + * transparently implemented using assembler macros, instead of needing separate + * cases depending on toolchain support. + * + * Simple usage example: + * __asm__ __volatile__("parse_r __rt, %0\n\t" + * "# di %0\n\t" + * ".word (0x41606000 | (__rt << 16))" + * : "=r" (status); + */ + +/* Match an individual register number and assign to \var */ +#define _IFC_REG(n) \ + ".ifc \\r, $r" #n "\n\t" \ + "\\var = " #n "\n\t" \ + ".endif\n\t" + +__asm__(".macro parse_r var r\n\t" + "\\var = -1\n\t" + _IFC_REG(0) _IFC_REG(1) _IFC_REG(2) _IFC_REG(3) + _IFC_REG(4) _IFC_REG(5) _IFC_REG(6) _IFC_REG(7) + _IFC_REG(8) _IFC_REG(9) _IFC_REG(10) _IFC_REG(11) + _IFC_REG(12) _IFC_REG(13) _IFC_REG(14) _IFC_REG(15) + _IFC_REG(16) _IFC_REG(17) _IFC_REG(18) _IFC_REG(19) + _IFC_REG(20) _IFC_REG(21) _IFC_REG(22) _IFC_REG(23) + _IFC_REG(24) _IFC_REG(25) _IFC_REG(26) _IFC_REG(27) + _IFC_REG(28) _IFC_REG(29) _IFC_REG(30) _IFC_REG(31) + ".iflt \\var\n\t" + ".error \"Unable to parse register name \\r\"\n\t" + ".endif\n\t" + ".endm"); + +#undef _IFC_REG + +/* CPUCFG */ +#define read_cpucfg(reg) __cpucfg(reg) + +#endif /* !__ASSEMBLY__ */ + +/* LoongArch Registers */ +#define REG_ZERO 0x0 +#define REG_RA 0x1 +#define REG_TP 0x2 +#define REG_SP 0x3 +#define REG_A0 0x4 /* Reused as V0 for return value */ +#define REG_A1 0x5 /* Reused as V1 for return value */ +#define REG_A2 0x6 +#define REG_A3 0x7 +#define REG_A4 0x8 +#define REG_A5 0x9 +#define REG_A6 0xa +#define REG_A7 0xb +#define REG_T0 0xc +#define REG_T1 0xd +#define REG_T2 0xe +#define REG_T3 0xf +#define REG_T4 0x10 +#define REG_T5 0x11 +#define REG_T6 0x12 +#define REG_T7 0x13 +#define REG_T8 0x14 +#define REG_U0 0x15 /* Kernel uses it as percpu base */ +#define REG_FP 0x16 +#define REG_S0 0x17 +#define REG_S1 0x18 +#define REG_S2 0x19 +#define REG_S3 0x1a +#define REG_S4 0x1b +#define REG_S5 0x1c +#define REG_S6 0x1d +#define REG_S7 0x1e +#define REG_S8 0x1f + +/* Bit fields for CPUCFG registers */ +#define LOONGARCH_CPUCFG0 0x0 +#define CPUCFG0_PRID GENMASK(31, 0) + +#define LOONGARCH_CPUCFG1 0x1 +#define CPUCFG1_ISGR32 BIT(0) +#define CPUCFG1_ISGR64 BIT(1) +#define CPUCFG1_PAGING BIT(2) +#define CPUCFG1_IOCSR BIT(3) +#define CPUCFG1_PABITS GENMASK(11, 4) +#define CPUCFG1_VABITS GENMASK(19, 12) +#define CPUCFG1_UAL BIT(20) +#define CPUCFG1_RI BIT(21) +#define CPUCFG1_EP BIT(22) +#define CPUCFG1_RPLV BIT(23) +#define CPUCFG1_HUGEPG BIT(24) +#define CPUCFG1_CRC32 BIT(25) +#define CPUCFG1_MSGINT BIT(26) + +#define LOONGARCH_CPUCFG2 0x2 +#define CPUCFG2_FP BIT(0) +#define CPUCFG2_FPSP BIT(1) +#define CPUCFG2_FPDP BIT(2) +#define CPUCFG2_FPVERS GENMASK(5, 3) +#define CPUCFG2_LSX BIT(6) +#define CPUCFG2_LASX BIT(7) +#define CPUCFG2_COMPLEX BIT(8) +#define CPUCFG2_CRYPTO BIT(9) +#define CPUCFG2_LVZP BIT(10) +#define CPUCFG2_LVZVER GENMASK(13, 11) +#define CPUCFG2_LLFTP BIT(14) +#define CPUCFG2_LLFTPREV GENMASK(17, 15) +#define CPUCFG2_X86BT BIT(18) +#define CPUCFG2_ARMBT BIT(19) +#define CPUCFG2_MIPSBT BIT(20) +#define CPUCFG2_LSPW BIT(21) +#define CPUCFG2_LAM BIT(22) +#define CPUCFG2_PTW BIT(24) + +#define LOONGARCH_CPUCFG3 0x3 +#define CPUCFG3_CCDMA BIT(0) +#define CPUCFG3_SFB BIT(1) +#define CPUCFG3_UCACC BIT(2) +#define CPUCFG3_LLEXC BIT(3) +#define CPUCFG3_SCDLY BIT(4) +#define CPUCFG3_LLDBAR BIT(5) +#define CPUCFG3_ITLBT BIT(6) +#define CPUCFG3_ICACHET BIT(7) +#define CPUCFG3_SPW_LVL GENMASK(10, 8) +#define CPUCFG3_SPW_HG_HF BIT(11) +#define CPUCFG3_RVA BIT(12) +#define CPUCFG3_RVAMAX GENMASK(16, 13) + +#define LOONGARCH_CPUCFG4 0x4 +#define CPUCFG4_CCFREQ GENMASK(31, 0) + +#define LOONGARCH_CPUCFG5 0x5 +#define CPUCFG5_CCMUL GENMASK(15, 0) +#define CPUCFG5_CCDIV GENMASK(31, 16) + +#define LOONGARCH_CPUCFG6 0x6 +#define CPUCFG6_PMP BIT(0) +#define CPUCFG6_PAMVER GENMASK(3, 1) +#define CPUCFG6_PMNUM GENMASK(7, 4) +#define CPUCFG6_PMBITS GENMASK(13, 8) +#define CPUCFG6_UPM BIT(14) + +#define LOONGARCH_CPUCFG16 0x10 +#define CPUCFG16_L1_IUPRE BIT(0) +#define CPUCFG16_L1_IUUNIFY BIT(1) +#define CPUCFG16_L1_DPRE BIT(2) +#define CPUCFG16_L2_IUPRE BIT(3) +#define CPUCFG16_L2_IUUNIFY BIT(4) +#define CPUCFG16_L2_IUPRIV BIT(5) +#define CPUCFG16_L2_IUINCL BIT(6) +#define CPUCFG16_L2_DPRE BIT(7) +#define CPUCFG16_L2_DPRIV BIT(8) +#define CPUCFG16_L2_DINCL BIT(9) +#define CPUCFG16_L3_IUPRE BIT(10) +#define CPUCFG16_L3_IUUNIFY BIT(11) +#define CPUCFG16_L3_IUPRIV BIT(12) +#define CPUCFG16_L3_IUINCL BIT(13) +#define CPUCFG16_L3_DPRE BIT(14) +#define CPUCFG16_L3_DPRIV BIT(15) +#define CPUCFG16_L3_DINCL BIT(16) + +#define LOONGARCH_CPUCFG17 0x11 +#define LOONGARCH_CPUCFG18 0x12 +#define LOONGARCH_CPUCFG19 0x13 +#define LOONGARCH_CPUCFG20 0x14 +#define CPUCFG_CACHE_WAYS_M GENMASK(15, 0) +#define CPUCFG_CACHE_SETS_M GENMASK(23, 16) +#define CPUCFG_CACHE_LSIZE_M GENMASK(30, 24) +#define CPUCFG_CACHE_WAYS 0 +#define CPUCFG_CACHE_SETS 16 +#define CPUCFG_CACHE_LSIZE 24 + +#define LOONGARCH_CPUCFG48 0x30 +#define CPUCFG48_MCSR_LCK BIT(0) +#define CPUCFG48_NAP_EN BIT(1) +#define CPUCFG48_VFPU_CG BIT(2) +#define CPUCFG48_RAM_CG BIT(3) + +#ifndef __ASSEMBLY__ + +/* CSR */ +#define csr_read32(reg) __csrrd_w(reg) +#define csr_read64(reg) __csrrd_d(reg) +#define csr_write32(val, reg) __csrwr_w(val, reg) +#define csr_write64(val, reg) __csrwr_d(val, reg) +#define csr_xchg32(val, mask, reg) __csrxchg_w(val, mask, reg) +#define csr_xchg64(val, mask, reg) __csrxchg_d(val, mask, reg) + +/* IOCSR */ +#define iocsr_read32(reg) __iocsrrd_w(reg) +#define iocsr_read64(reg) __iocsrrd_d(reg) +#define iocsr_write8(val, reg) __iocsrwr_b(val, reg) +#define iocsr_write32(val, reg) __iocsrwr_w(val, reg) +#define iocsr_write64(val, reg) __iocsrwr_d(val, reg) + +/* GCSR */ +#define gcsr_read(csr) \ +({ \ + register unsigned long __v = 0; \ + __asm__ __volatile__ ( \ + "parse_r __reg, %[val] \n\t" \ + ".word 0x5 << 24 | %[reg] << 10 | 0 << 5 | __reg \n\t" \ + : [val] "+r" (__v) \ + : [reg] "i" (csr) \ + : "memory"); \ + __v; \ +}) + +#define gcsr_write(v, csr) \ +({ \ + register unsigned long __v = v; \ + __asm__ __volatile__ ( \ + "parse_r __reg, %[val] \n\t" \ + ".word 0x5 << 24 | %[reg] << 10 | 1 << 5 | __reg \n\t" \ + : [val] "+r" (__v) \ + : [reg] "i" (csr) \ + : "memory"); \ +}) + +#define gcsr_xchg(v, m, csr) \ +({ \ + register unsigned long __v = v; \ + __asm__ __volatile__ ( \ + "parse_r __rd, %[val] \n\t" \ + "parse_r __rj, %[mask] \n\t" \ + ".word 0x5 << 24 | %[reg] << 10 | __rj << 5 | __rd \n\t" \ + : [val] "+r" (__v) \ + : [mask] "r" (m), [reg] "i" (csr) \ + : "memory"); \ + __v; \ +}) + +#endif /* !__ASSEMBLY__ */ + +/* CSR register number */ + +/* Basic CSR registers */ +#define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */ +#define CSR_CRMD_WE_SHIFT 9 +#define CSR_CRMD_WE (_ULCAST_(0x1) << CSR_CRMD_WE_SHIFT) +#define CSR_CRMD_DACM_SHIFT 7 +#define CSR_CRMD_DACM_WIDTH 2 +#define CSR_CRMD_DACM (_ULCAST_(0x3) << CSR_CRMD_DACM_SHIFT) +#define CSR_CRMD_DACF_SHIFT 5 +#define CSR_CRMD_DACF_WIDTH 2 +#define CSR_CRMD_DACF (_ULCAST_(0x3) << CSR_CRMD_DACF_SHIFT) +#define CSR_CRMD_PG_SHIFT 4 +#define CSR_CRMD_PG (_ULCAST_(0x1) << CSR_CRMD_PG_SHIFT) +#define CSR_CRMD_DA_SHIFT 3 +#define CSR_CRMD_DA (_ULCAST_(0x1) << CSR_CRMD_DA_SHIFT) +#define CSR_CRMD_IE_SHIFT 2 +#define CSR_CRMD_IE (_ULCAST_(0x1) << CSR_CRMD_IE_SHIFT) +#define CSR_CRMD_PLV_SHIFT 0 +#define CSR_CRMD_PLV_WIDTH 2 +#define CSR_CRMD_PLV (_ULCAST_(0x3) << CSR_CRMD_PLV_SHIFT) + +#define PLV_KERN 0 +#define PLV_USER 3 +#define PLV_MASK 0x3 + +#define LOONGARCH_CSR_PRMD 0x1 /* Prev-exception mode info */ +#define CSR_PRMD_PWE_SHIFT 3 +#define CSR_PRMD_PWE (_ULCAST_(0x1) << CSR_PRMD_PWE_SHIFT) +#define CSR_PRMD_PIE_SHIFT 2 +#define CSR_PRMD_PIE (_ULCAST_(0x1) << CSR_PRMD_PIE_SHIFT) +#define CSR_PRMD_PPLV_SHIFT 0 +#define CSR_PRMD_PPLV_WIDTH 2 +#define CSR_PRMD_PPLV (_ULCAST_(0x3) << CSR_PRMD_PPLV_SHIFT) + +#define LOONGARCH_CSR_EUEN 0x2 /* Extended unit enable */ +#define CSR_EUEN_LBTEN_SHIFT 3 +#define CSR_EUEN_LBTEN (_ULCAST_(0x1) << CSR_EUEN_LBTEN_SHIFT) +#define CSR_EUEN_LASXEN_SHIFT 2 +#define CSR_EUEN_LASXEN (_ULCAST_(0x1) << CSR_EUEN_LASXEN_SHIFT) +#define CSR_EUEN_LSXEN_SHIFT 1 +#define CSR_EUEN_LSXEN (_ULCAST_(0x1) << CSR_EUEN_LSXEN_SHIFT) +#define CSR_EUEN_FPEN_SHIFT 0 +#define CSR_EUEN_FPEN (_ULCAST_(0x1) << CSR_EUEN_FPEN_SHIFT) + +#define LOONGARCH_CSR_MISC 0x3 /* Misc config */ + +#define LOONGARCH_CSR_ECFG 0x4 /* Exception config */ +#define CSR_ECFG_VS_SHIFT 16 +#define CSR_ECFG_VS_WIDTH 3 +#define CSR_ECFG_VS (_ULCAST_(0x7) << CSR_ECFG_VS_SHIFT) +#define CSR_ECFG_IM_SHIFT 0 +#define CSR_ECFG_IM_WIDTH 14 +#define CSR_ECFG_IM (_ULCAST_(0x3fff) << CSR_ECFG_IM_SHIFT) + +#define LOONGARCH_CSR_ESTAT 0x5 /* Exception status */ +#define CSR_ESTAT_ESUBCODE_SHIFT 22 +#define CSR_ESTAT_ESUBCODE_WIDTH 9 +#define CSR_ESTAT_ESUBCODE (_ULCAST_(0x1ff) << CSR_ESTAT_ESUBCODE_SHIFT) +#define CSR_ESTAT_EXC_SHIFT 16 +#define CSR_ESTAT_EXC_WIDTH 6 +#define CSR_ESTAT_EXC (_ULCAST_(0x3f) << CSR_ESTAT_EXC_SHIFT) +#define CSR_ESTAT_IS_SHIFT 0 +#define CSR_ESTAT_IS_WIDTH 14 +#define CSR_ESTAT_IS (_ULCAST_(0x3fff) << CSR_ESTAT_IS_SHIFT) + +#define LOONGARCH_CSR_ERA 0x6 /* ERA */ + +#define LOONGARCH_CSR_BADV 0x7 /* Bad virtual address */ + +#define LOONGARCH_CSR_BADI 0x8 /* Bad instruction */ + +#define LOONGARCH_CSR_EENTRY 0xc /* Exception entry */ + +/* TLB related CSR registers */ +#define LOONGARCH_CSR_TLBIDX 0x10 /* TLB Index, EHINV, PageSize, NP */ +#define CSR_TLBIDX_EHINV_SHIFT 31 +#define CSR_TLBIDX_EHINV (_ULCAST_(1) << CSR_TLBIDX_EHINV_SHIFT) +#define CSR_TLBIDX_PS_SHIFT 24 +#define CSR_TLBIDX_PS_WIDTH 6 +#define CSR_TLBIDX_PS (_ULCAST_(0x3f) << CSR_TLBIDX_PS_SHIFT) +#define CSR_TLBIDX_IDX_SHIFT 0 +#define CSR_TLBIDX_IDX_WIDTH 12 +#define CSR_TLBIDX_IDX (_ULCAST_(0xfff) << CSR_TLBIDX_IDX_SHIFT) +#define CSR_TLBIDX_SIZEM 0x3f000000 +#define CSR_TLBIDX_SIZE CSR_TLBIDX_PS_SHIFT +#define CSR_TLBIDX_IDXM 0xfff +#define CSR_INVALID_ENTRY(e) (CSR_TLBIDX_EHINV | e) + +#define LOONGARCH_CSR_TLBEHI 0x11 /* TLB EntryHi */ + +#define LOONGARCH_CSR_TLBELO0 0x12 /* TLB EntryLo0 */ +#define CSR_TLBLO0_RPLV_SHIFT 63 +#define CSR_TLBLO0_RPLV (_ULCAST_(0x1) << CSR_TLBLO0_RPLV_SHIFT) +#define CSR_TLBLO0_NX_SHIFT 62 +#define CSR_TLBLO0_NX (_ULCAST_(0x1) << CSR_TLBLO0_NX_SHIFT) +#define CSR_TLBLO0_NR_SHIFT 61 +#define CSR_TLBLO0_NR (_ULCAST_(0x1) << CSR_TLBLO0_NR_SHIFT) +#define CSR_TLBLO0_PFN_SHIFT 12 +#define CSR_TLBLO0_PFN_WIDTH 36 +#define CSR_TLBLO0_PFN (_ULCAST_(0xfffffffff) << CSR_TLBLO0_PFN_SHIFT) +#define CSR_TLBLO0_GLOBAL_SHIFT 6 +#define CSR_TLBLO0_GLOBAL (_ULCAST_(0x1) << CSR_TLBLO0_GLOBAL_SHIFT) +#define CSR_TLBLO0_CCA_SHIFT 4 +#define CSR_TLBLO0_CCA_WIDTH 2 +#define CSR_TLBLO0_CCA (_ULCAST_(0x3) << CSR_TLBLO0_CCA_SHIFT) +#define CSR_TLBLO0_PLV_SHIFT 2 +#define CSR_TLBLO0_PLV_WIDTH 2 +#define CSR_TLBLO0_PLV (_ULCAST_(0x3) << CSR_TLBLO0_PLV_SHIFT) +#define CSR_TLBLO0_WE_SHIFT 1 +#define CSR_TLBLO0_WE (_ULCAST_(0x1) << CSR_TLBLO0_WE_SHIFT) +#define CSR_TLBLO0_V_SHIFT 0 +#define CSR_TLBLO0_V (_ULCAST_(0x1) << CSR_TLBLO0_V_SHIFT) + +#define LOONGARCH_CSR_TLBELO1 0x13 /* TLB EntryLo1 */ +#define CSR_TLBLO1_RPLV_SHIFT 63 +#define CSR_TLBLO1_RPLV (_ULCAST_(0x1) << CSR_TLBLO1_RPLV_SHIFT) +#define CSR_TLBLO1_NX_SHIFT 62 +#define CSR_TLBLO1_NX (_ULCAST_(0x1) << CSR_TLBLO1_NX_SHIFT) +#define CSR_TLBLO1_NR_SHIFT 61 +#define CSR_TLBLO1_NR (_ULCAST_(0x1) << CSR_TLBLO1_NR_SHIFT) +#define CSR_TLBLO1_PFN_SHIFT 12 +#define CSR_TLBLO1_PFN_WIDTH 36 +#define CSR_TLBLO1_PFN (_ULCAST_(0xfffffffff) << CSR_TLBLO1_PFN_SHIFT) +#define CSR_TLBLO1_GLOBAL_SHIFT 6 +#define CSR_TLBLO1_GLOBAL (_ULCAST_(0x1) << CSR_TLBLO1_GLOBAL_SHIFT) +#define CSR_TLBLO1_CCA_SHIFT 4 +#define CSR_TLBLO1_CCA_WIDTH 2 +#define CSR_TLBLO1_CCA (_ULCAST_(0x3) << CSR_TLBLO1_CCA_SHIFT) +#define CSR_TLBLO1_PLV_SHIFT 2 +#define CSR_TLBLO1_PLV_WIDTH 2 +#define CSR_TLBLO1_PLV (_ULCAST_(0x3) << CSR_TLBLO1_PLV_SHIFT) +#define CSR_TLBLO1_WE_SHIFT 1 +#define CSR_TLBLO1_WE (_ULCAST_(0x1) << CSR_TLBLO1_WE_SHIFT) +#define CSR_TLBLO1_V_SHIFT 0 +#define CSR_TLBLO1_V (_ULCAST_(0x1) << CSR_TLBLO1_V_SHIFT) + +#define LOONGARCH_CSR_GTLBC 0x15 /* Guest TLB control */ +#define CSR_GTLBC_TGID_SHIFT 16 +#define CSR_GTLBC_TGID_WIDTH 8 +#define CSR_GTLBC_TGID (_ULCAST_(0xff) << CSR_GTLBC_TGID_SHIFT) +#define CSR_GTLBC_TOTI_SHIFT 13 +#define CSR_GTLBC_TOTI (_ULCAST_(0x1) << CSR_GTLBC_TOTI_SHIFT) +#define CSR_GTLBC_USETGID_SHIFT 12 +#define CSR_GTLBC_USETGID (_ULCAST_(0x1) << CSR_GTLBC_USETGID_SHIFT) +#define CSR_GTLBC_GMTLBSZ_SHIFT 0 +#define CSR_GTLBC_GMTLBSZ_WIDTH 6 +#define CSR_GTLBC_GMTLBSZ (_ULCAST_(0x3f) << CSR_GTLBC_GMTLBSZ_SHIFT) + +#define LOONGARCH_CSR_TRGP 0x16 /* TLBR read guest info */ +#define CSR_TRGP_RID_SHIFT 16 +#define CSR_TRGP_RID_WIDTH 8 +#define CSR_TRGP_RID (_ULCAST_(0xff) << CSR_TRGP_RID_SHIFT) +#define CSR_TRGP_GTLB_SHIFT 0 +#define CSR_TRGP_GTLB (1 << CSR_TRGP_GTLB_SHIFT) + +#define LOONGARCH_CSR_ASID 0x18 /* ASID */ +#define CSR_ASID_BIT_SHIFT 16 /* ASIDBits */ +#define CSR_ASID_BIT_WIDTH 8 +#define CSR_ASID_BIT (_ULCAST_(0xff) << CSR_ASID_BIT_SHIFT) +#define CSR_ASID_ASID_SHIFT 0 +#define CSR_ASID_ASID_WIDTH 10 +#define CSR_ASID_ASID (_ULCAST_(0x3ff) << CSR_ASID_ASID_SHIFT) + +#define LOONGARCH_CSR_PGDL 0x19 /* Page table base address when VA[VALEN-1] = 0 */ + +#define LOONGARCH_CSR_PGDH 0x1a /* Page table base address when VA[VALEN-1] = 1 */ + +#define LOONGARCH_CSR_PGD 0x1b /* Page table base */ + +#define LOONGARCH_CSR_PWCTL0 0x1c /* PWCtl0 */ +#define CSR_PWCTL0_PTEW_SHIFT 30 +#define CSR_PWCTL0_PTEW_WIDTH 2 +#define CSR_PWCTL0_PTEW (_ULCAST_(0x3) << CSR_PWCTL0_PTEW_SHIFT) +#define CSR_PWCTL0_DIR1WIDTH_SHIFT 25 +#define CSR_PWCTL0_DIR1WIDTH_WIDTH 5 +#define CSR_PWCTL0_DIR1WIDTH (_ULCAST_(0x1f) << CSR_PWCTL0_DIR1WIDTH_SHIFT) +#define CSR_PWCTL0_DIR1BASE_SHIFT 20 +#define CSR_PWCTL0_DIR1BASE_WIDTH 5 +#define CSR_PWCTL0_DIR1BASE (_ULCAST_(0x1f) << CSR_PWCTL0_DIR1BASE_SHIFT) +#define CSR_PWCTL0_DIR0WIDTH_SHIFT 15 +#define CSR_PWCTL0_DIR0WIDTH_WIDTH 5 +#define CSR_PWCTL0_DIR0WIDTH (_ULCAST_(0x1f) << CSR_PWCTL0_DIR0WIDTH_SHIFT) +#define CSR_PWCTL0_DIR0BASE_SHIFT 10 +#define CSR_PWCTL0_DIR0BASE_WIDTH 5 +#define CSR_PWCTL0_DIR0BASE (_ULCAST_(0x1f) << CSR_PWCTL0_DIR0BASE_SHIFT) +#define CSR_PWCTL0_PTWIDTH_SHIFT 5 +#define CSR_PWCTL0_PTWIDTH_WIDTH 5 +#define CSR_PWCTL0_PTWIDTH (_ULCAST_(0x1f) << CSR_PWCTL0_PTWIDTH_SHIFT) +#define CSR_PWCTL0_PTBASE_SHIFT 0 +#define CSR_PWCTL0_PTBASE_WIDTH 5 +#define CSR_PWCTL0_PTBASE (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT) + +#define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */ +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW_WIDTH 1 +#define CSR_PWCTL1_PTW (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT) +#define CSR_PWCTL1_DIR3WIDTH_SHIFT 18 +#define CSR_PWCTL1_DIR3WIDTH_WIDTH 5 +#define CSR_PWCTL1_DIR3WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT) +#define CSR_PWCTL1_DIR3BASE_SHIFT 12 +#define CSR_PWCTL1_DIR3BASE_WIDTH 5 +#define CSR_PWCTL1_DIR3BASE (_ULCAST_(0x1f) << CSR_PWCTL0_DIR3BASE_SHIFT) +#define CSR_PWCTL1_DIR2WIDTH_SHIFT 6 +#define CSR_PWCTL1_DIR2WIDTH_WIDTH 5 +#define CSR_PWCTL1_DIR2WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR2WIDTH_SHIFT) +#define CSR_PWCTL1_DIR2BASE_SHIFT 0 +#define CSR_PWCTL1_DIR2BASE_WIDTH 5 +#define CSR_PWCTL1_DIR2BASE (_ULCAST_(0x1f) << CSR_PWCTL0_DIR2BASE_SHIFT) + +#define LOONGARCH_CSR_STLBPGSIZE 0x1e +#define CSR_STLBPGSIZE_PS_WIDTH 6 +#define CSR_STLBPGSIZE_PS (_ULCAST_(0x3f)) + +#define LOONGARCH_CSR_RVACFG 0x1f +#define CSR_RVACFG_RDVA_WIDTH 4 +#define CSR_RVACFG_RDVA (_ULCAST_(0xf)) + +/* Config CSR registers */ +#define LOONGARCH_CSR_CPUID 0x20 /* CPU core id */ +#define CSR_CPUID_COREID_WIDTH 9 +#define CSR_CPUID_COREID _ULCAST_(0x1ff) + +#define LOONGARCH_CSR_PRCFG1 0x21 /* Config1 */ +#define CSR_CONF1_VSMAX_SHIFT 12 +#define CSR_CONF1_VSMAX_WIDTH 3 +#define CSR_CONF1_VSMAX (_ULCAST_(7) << CSR_CONF1_VSMAX_SHIFT) +#define CSR_CONF1_TMRBITS_SHIFT 4 +#define CSR_CONF1_TMRBITS_WIDTH 8 +#define CSR_CONF1_TMRBITS (_ULCAST_(0xff) << CSR_CONF1_TMRBITS_SHIFT) +#define CSR_CONF1_KSNUM_WIDTH 4 +#define CSR_CONF1_KSNUM _ULCAST_(0xf) + +#define LOONGARCH_CSR_PRCFG2 0x22 /* Config2 */ +#define CSR_CONF2_PGMASK_SUPP 0x3ffff000 + +#define LOONGARCH_CSR_PRCFG3 0x23 /* Config3 */ +#define CSR_CONF3_STLBIDX_SHIFT 20 +#define CSR_CONF3_STLBIDX_WIDTH 6 +#define CSR_CONF3_STLBIDX (_ULCAST_(0x3f) << CSR_CONF3_STLBIDX_SHIFT) +#define CSR_CONF3_STLBWAYS_SHIFT 12 +#define CSR_CONF3_STLBWAYS_WIDTH 8 +#define CSR_CONF3_STLBWAYS (_ULCAST_(0xff) << CSR_CONF3_STLBWAYS_SHIFT) +#define CSR_CONF3_MTLBSIZE_SHIFT 4 +#define CSR_CONF3_MTLBSIZE_WIDTH 8 +#define CSR_CONF3_MTLBSIZE (_ULCAST_(0xff) << CSR_CONF3_MTLBSIZE_SHIFT) +#define CSR_CONF3_TLBTYPE_SHIFT 0 +#define CSR_CONF3_TLBTYPE_WIDTH 4 +#define CSR_CONF3_TLBTYPE (_ULCAST_(0xf) << CSR_CONF3_TLBTYPE_SHIFT) + +/* KSave registers */ +#define LOONGARCH_CSR_KS0 0x30 +#define LOONGARCH_CSR_KS1 0x31 +#define LOONGARCH_CSR_KS2 0x32 +#define LOONGARCH_CSR_KS3 0x33 +#define LOONGARCH_CSR_KS4 0x34 +#define LOONGARCH_CSR_KS5 0x35 +#define LOONGARCH_CSR_KS6 0x36 +#define LOONGARCH_CSR_KS7 0x37 +#define LOONGARCH_CSR_KS8 0x38 + +/* Exception allocated KS0, KS1 and KS2 statically */ +#define EXCEPTION_KS0 LOONGARCH_CSR_KS0 +#define EXCEPTION_KS1 LOONGARCH_CSR_KS1 +#define EXCEPTION_KS2 LOONGARCH_CSR_KS2 +#define EXC_KSAVE_MASK (1 << 0 | 1 << 1 | 1 << 2) + +/* Percpu-data base allocated KS3 statically */ +#define PERCPU_BASE_KS LOONGARCH_CSR_KS3 +#define PERCPU_KSAVE_MASK (1 << 3) + +/* KVM allocated KS4 and KS5 statically */ +#define KVM_VCPU_KS LOONGARCH_CSR_KS4 +#define KVM_TEMP_KS LOONGARCH_CSR_KS5 +#define KVM_KSAVE_MASK (1 << 4 | 1 << 5) + +/* Timer registers */ +#define LOONGARCH_CSR_TMID 0x40 /* Timer ID */ + +#define LOONGARCH_CSR_TCFG 0x41 /* Timer config */ +#define CSR_TCFG_VAL_SHIFT 2 +#define CSR_TCFG_VAL_WIDTH 48 +#define CSR_TCFG_VAL (_ULCAST_(0x3fffffffffff) << CSR_TCFG_VAL_SHIFT) +#define CSR_TCFG_PERIOD_SHIFT 1 +#define CSR_TCFG_PERIOD (_ULCAST_(0x1) << CSR_TCFG_PERIOD_SHIFT) +#define CSR_TCFG_EN (_ULCAST_(0x1)) + +#define LOONGARCH_CSR_TVAL 0x42 /* Timer value */ + +#define LOONGARCH_CSR_CNTC 0x43 /* Timer offset */ + +#define LOONGARCH_CSR_TINTCLR 0x44 /* Timer interrupt clear */ +#define CSR_TINTCLR_TI_SHIFT 0 +#define CSR_TINTCLR_TI (1 << CSR_TINTCLR_TI_SHIFT) + +/* Guest registers */ +#define LOONGARCH_CSR_GSTAT 0x50 /* Guest status */ +#define CSR_GSTAT_GID_SHIFT 16 +#define CSR_GSTAT_GID_WIDTH 8 +#define CSR_GSTAT_GID (_ULCAST_(0xff) << CSR_GSTAT_GID_SHIFT) +#define CSR_GSTAT_GIDBIT_SHIFT 4 +#define CSR_GSTAT_GIDBIT_WIDTH 6 +#define CSR_GSTAT_GIDBIT (_ULCAST_(0x3f) << CSR_GSTAT_GIDBIT_SHIFT) +#define CSR_GSTAT_PVM_SHIFT 1 +#define CSR_GSTAT_PVM (_ULCAST_(0x1) << CSR_GSTAT_PVM_SHIFT) +#define CSR_GSTAT_VM_SHIFT 0 +#define CSR_GSTAT_VM (_ULCAST_(0x1) << CSR_GSTAT_VM_SHIFT) + +#define LOONGARCH_CSR_GCFG 0x51 /* Guest config */ +#define CSR_GCFG_GPERF_SHIFT 24 +#define CSR_GCFG_GPERF_WIDTH 3 +#define CSR_GCFG_GPERF (_ULCAST_(0x7) << CSR_GCFG_GPERF_SHIFT) +#define CSR_GCFG_GCI_SHIFT 20 +#define CSR_GCFG_GCI_WIDTH 2 +#define CSR_GCFG_GCI (_ULCAST_(0x3) << CSR_GCFG_GCI_SHIFT) +#define CSR_GCFG_GCI_ALL (_ULCAST_(0x0) << CSR_GCFG_GCI_SHIFT) +#define CSR_GCFG_GCI_HIT (_ULCAST_(0x1) << CSR_GCFG_GCI_SHIFT) +#define CSR_GCFG_GCI_SECURE (_ULCAST_(0x2) << CSR_GCFG_GCI_SHIFT) +#define CSR_GCFG_GCIP_SHIFT 16 +#define CSR_GCFG_GCIP (_ULCAST_(0xf) << CSR_GCFG_GCIP_SHIFT) +#define CSR_GCFG_GCIP_ALL (_ULCAST_(0x1) << CSR_GCFG_GCIP_SHIFT) +#define CSR_GCFG_GCIP_HIT (_ULCAST_(0x1) << (CSR_GCFG_GCIP_SHIFT + 1)) +#define CSR_GCFG_GCIP_SECURE (_ULCAST_(0x1) << (CSR_GCFG_GCIP_SHIFT + 2)) +#define CSR_GCFG_TORU_SHIFT 15 +#define CSR_GCFG_TORU (_ULCAST_(0x1) << CSR_GCFG_TORU_SHIFT) +#define CSR_GCFG_TORUP_SHIFT 14 +#define CSR_GCFG_TORUP (_ULCAST_(0x1) << CSR_GCFG_TORUP_SHIFT) +#define CSR_GCFG_TOP_SHIFT 13 +#define CSR_GCFG_TOP (_ULCAST_(0x1) << CSR_GCFG_TOP_SHIFT) +#define CSR_GCFG_TOPP_SHIFT 12 +#define CSR_GCFG_TOPP (_ULCAST_(0x1) << CSR_GCFG_TOPP_SHIFT) +#define CSR_GCFG_TOE_SHIFT 11 +#define CSR_GCFG_TOE (_ULCAST_(0x1) << CSR_GCFG_TOE_SHIFT) +#define CSR_GCFG_TOEP_SHIFT 10 +#define CSR_GCFG_TOEP (_ULCAST_(0x1) << CSR_GCFG_TOEP_SHIFT) +#define CSR_GCFG_TIT_SHIFT 9 +#define CSR_GCFG_TIT (_ULCAST_(0x1) << CSR_GCFG_TIT_SHIFT) +#define CSR_GCFG_TITP_SHIFT 8 +#define CSR_GCFG_TITP (_ULCAST_(0x1) << CSR_GCFG_TITP_SHIFT) +#define CSR_GCFG_SIT_SHIFT 7 +#define CSR_GCFG_SIT (_ULCAST_(0x1) << CSR_GCFG_SIT_SHIFT) +#define CSR_GCFG_SITP_SHIFT 6 +#define CSR_GCFG_SITP (_ULCAST_(0x1) << CSR_GCFG_SITP_SHIFT) +#define CSR_GCFG_MATC_SHITF 4 +#define CSR_GCFG_MATC_WIDTH 2 +#define CSR_GCFG_MATC_MASK (_ULCAST_(0x3) << CSR_GCFG_MATC_SHITF) +#define CSR_GCFG_MATC_GUEST (_ULCAST_(0x0) << CSR_GCFG_MATC_SHITF) +#define CSR_GCFG_MATC_ROOT (_ULCAST_(0x1) << CSR_GCFG_MATC_SHITF) +#define CSR_GCFG_MATC_NEST (_ULCAST_(0x2) << CSR_GCFG_MATC_SHITF) +#define CSR_GCFG_MATP_SHITF 0 +#define CSR_GCFG_MATP_WIDTH 4 +#define CSR_GCFG_MATP_MASK (_ULCAST_(0x3) << CSR_GCFG_MATP_SHITF) +#define CSR_GCFG_MATP_GUEST (_ULCAST_(0x0) << CSR_GCFG_MATP_SHITF) +#define CSR_GCFG_MATP_ROOT (_ULCAST_(0x1) << CSR_GCFG_MATP_SHITF) +#define CSR_GCFG_MATP_NEST (_ULCAST_(0x2) << CSR_GCFG_MATP_SHITF) + +#define LOONGARCH_CSR_GINTC 0x52 /* Guest interrupt control */ +#define CSR_GINTC_HC_SHIFT 16 +#define CSR_GINTC_HC_WIDTH 8 +#define CSR_GINTC_HC (_ULCAST_(0xff) << CSR_GINTC_HC_SHIFT) +#define CSR_GINTC_PIP_SHIFT 8 +#define CSR_GINTC_PIP_WIDTH 8 +#define CSR_GINTC_PIP (_ULCAST_(0xff) << CSR_GINTC_PIP_SHIFT) +#define CSR_GINTC_VIP_SHIFT 0 +#define CSR_GINTC_VIP_WIDTH 8 +#define CSR_GINTC_VIP (_ULCAST_(0xff)) + +#define LOONGARCH_CSR_GCNTC 0x53 /* Guest timer offset */ + +/* LLBCTL register */ +#define LOONGARCH_CSR_LLBCTL 0x60 /* LLBit control */ +#define CSR_LLBCTL_ROLLB_SHIFT 0 +#define CSR_LLBCTL_ROLLB (_ULCAST_(1) << CSR_LLBCTL_ROLLB_SHIFT) +#define CSR_LLBCTL_WCLLB_SHIFT 1 +#define CSR_LLBCTL_WCLLB (_ULCAST_(1) << CSR_LLBCTL_WCLLB_SHIFT) +#define CSR_LLBCTL_KLO_SHIFT 2 +#define CSR_LLBCTL_KLO (_ULCAST_(1) << CSR_LLBCTL_KLO_SHIFT) + +/* Implement dependent */ +#define LOONGARCH_CSR_IMPCTL1 0x80 /* Loongson config1 */ +#define CSR_MISPEC_SHIFT 20 +#define CSR_MISPEC_WIDTH 8 +#define CSR_MISPEC (_ULCAST_(0xff) << CSR_MISPEC_SHIFT) +#define CSR_SSEN_SHIFT 18 +#define CSR_SSEN (_ULCAST_(1) << CSR_SSEN_SHIFT) +#define CSR_SCRAND_SHIFT 17 +#define CSR_SCRAND (_ULCAST_(1) << CSR_SCRAND_SHIFT) +#define CSR_LLEXCL_SHIFT 16 +#define CSR_LLEXCL (_ULCAST_(1) << CSR_LLEXCL_SHIFT) +#define CSR_DISVC_SHIFT 15 +#define CSR_DISVC (_ULCAST_(1) << CSR_DISVC_SHIFT) +#define CSR_VCLRU_SHIFT 14 +#define CSR_VCLRU (_ULCAST_(1) << CSR_VCLRU_SHIFT) +#define CSR_DCLRU_SHIFT 13 +#define CSR_DCLRU (_ULCAST_(1) << CSR_DCLRU_SHIFT) +#define CSR_FASTLDQ_SHIFT 12 +#define CSR_FASTLDQ (_ULCAST_(1) << CSR_FASTLDQ_SHIFT) +#define CSR_USERCAC_SHIFT 11 +#define CSR_USERCAC (_ULCAST_(1) << CSR_USERCAC_SHIFT) +#define CSR_ANTI_MISPEC_SHIFT 10 +#define CSR_ANTI_MISPEC (_ULCAST_(1) << CSR_ANTI_MISPEC_SHIFT) +#define CSR_AUTO_FLUSHSFB_SHIFT 9 +#define CSR_AUTO_FLUSHSFB (_ULCAST_(1) << CSR_AUTO_FLUSHSFB_SHIFT) +#define CSR_STFILL_SHIFT 8 +#define CSR_STFILL (_ULCAST_(1) << CSR_STFILL_SHIFT) +#define CSR_LIFEP_SHIFT 7 +#define CSR_LIFEP (_ULCAST_(1) << CSR_LIFEP_SHIFT) +#define CSR_LLSYNC_SHIFT 6 +#define CSR_LLSYNC (_ULCAST_(1) << CSR_LLSYNC_SHIFT) +#define CSR_BRBTDIS_SHIFT 5 +#define CSR_BRBTDIS (_ULCAST_(1) << CSR_BRBTDIS_SHIFT) +#define CSR_RASDIS_SHIFT 4 +#define CSR_RASDIS (_ULCAST_(1) << CSR_RASDIS_SHIFT) +#define CSR_STPRE_SHIFT 2 +#define CSR_STPRE_WIDTH 2 +#define CSR_STPRE (_ULCAST_(3) << CSR_STPRE_SHIFT) +#define CSR_INSTPRE_SHIFT 1 +#define CSR_INSTPRE (_ULCAST_(1) << CSR_INSTPRE_SHIFT) +#define CSR_DATAPRE_SHIFT 0 +#define CSR_DATAPRE (_ULCAST_(1) << CSR_DATAPRE_SHIFT) + +#define LOONGARCH_CSR_IMPCTL2 0x81 /* Loongson config2 */ +#define CSR_FLUSH_MTLB_SHIFT 0 +#define CSR_FLUSH_MTLB (_ULCAST_(1) << CSR_FLUSH_MTLB_SHIFT) +#define CSR_FLUSH_STLB_SHIFT 1 +#define CSR_FLUSH_STLB (_ULCAST_(1) << CSR_FLUSH_STLB_SHIFT) +#define CSR_FLUSH_DTLB_SHIFT 2 +#define CSR_FLUSH_DTLB (_ULCAST_(1) << CSR_FLUSH_DTLB_SHIFT) +#define CSR_FLUSH_ITLB_SHIFT 3 +#define CSR_FLUSH_ITLB (_ULCAST_(1) << CSR_FLUSH_ITLB_SHIFT) +#define CSR_FLUSH_BTAC_SHIFT 4 +#define CSR_FLUSH_BTAC (_ULCAST_(1) << CSR_FLUSH_BTAC_SHIFT) + +#define LOONGARCH_CSR_GNMI 0x82 + +/* TLB Refill registers */ +#define LOONGARCH_CSR_TLBRENTRY 0x88 /* TLB refill exception entry */ +#define LOONGARCH_CSR_TLBRBADV 0x89 /* TLB refill badvaddr */ +#define LOONGARCH_CSR_TLBRERA 0x8a /* TLB refill ERA */ +#define LOONGARCH_CSR_TLBRSAVE 0x8b /* KSave for TLB refill exception */ +#define LOONGARCH_CSR_TLBRELO0 0x8c /* TLB refill entrylo0 */ +#define LOONGARCH_CSR_TLBRELO1 0x8d /* TLB refill entrylo1 */ +#define LOONGARCH_CSR_TLBREHI 0x8e /* TLB refill entryhi */ +#define CSR_TLBREHI_PS_SHIFT 0 +#define CSR_TLBREHI_PS (_ULCAST_(0x3f) << CSR_TLBREHI_PS_SHIFT) +#define LOONGARCH_CSR_TLBRPRMD 0x8f /* TLB refill mode info */ + +/* Machine Error registers */ +#define LOONGARCH_CSR_MERRCTL 0x90 /* MERRCTL */ +#define LOONGARCH_CSR_MERRINFO1 0x91 /* MError info1 */ +#define LOONGARCH_CSR_MERRINFO2 0x92 /* MError info2 */ +#define LOONGARCH_CSR_MERRENTRY 0x93 /* MError exception entry */ +#define LOONGARCH_CSR_MERRERA 0x94 /* MError exception ERA */ +#define LOONGARCH_CSR_MERRSAVE 0x95 /* KSave for machine error exception */ + +#define LOONGARCH_CSR_CTAG 0x98 /* TagLo + TagHi */ + +#define LOONGARCH_CSR_PRID 0xc0 + +/* Shadow MCSR : 0xc0 ~ 0xff */ +#define LOONGARCH_CSR_MCSR0 0xc0 /* CPUCFG0 and CPUCFG1 */ +#define MCSR0_INT_IMPL_SHIFT 58 +#define MCSR0_INT_IMPL 0 +#define MCSR0_IOCSR_BRD_SHIFT 57 +#define MCSR0_IOCSR_BRD (_ULCAST_(1) << MCSR0_IOCSR_BRD_SHIFT) +#define MCSR0_HUGEPG_SHIFT 56 +#define MCSR0_HUGEPG (_ULCAST_(1) << MCSR0_HUGEPG_SHIFT) +#define MCSR0_RPLMTLB_SHIFT 55 +#define MCSR0_RPLMTLB (_ULCAST_(1) << MCSR0_RPLMTLB_SHIFT) +#define MCSR0_EP_SHIFT 54 +#define MCSR0_EP (_ULCAST_(1) << MCSR0_EP_SHIFT) +#define MCSR0_RI_SHIFT 53 +#define MCSR0_RI (_ULCAST_(1) << MCSR0_RI_SHIFT) +#define MCSR0_UAL_SHIFT 52 +#define MCSR0_UAL (_ULCAST_(1) << MCSR0_UAL_SHIFT) +#define MCSR0_VABIT_SHIFT 44 +#define MCSR0_VABIT_WIDTH 8 +#define MCSR0_VABIT (_ULCAST_(0xff) << MCSR0_VABIT_SHIFT) +#define VABIT_DEFAULT 0x2f +#define MCSR0_PABIT_SHIFT 36 +#define MCSR0_PABIT_WIDTH 8 +#define MCSR0_PABIT (_ULCAST_(0xff) << MCSR0_PABIT_SHIFT) +#define PABIT_DEFAULT 0x2f +#define MCSR0_IOCSR_SHIFT 35 +#define MCSR0_IOCSR (_ULCAST_(1) << MCSR0_IOCSR_SHIFT) +#define MCSR0_PAGING_SHIFT 34 +#define MCSR0_PAGING (_ULCAST_(1) << MCSR0_PAGING_SHIFT) +#define MCSR0_GR64_SHIFT 33 +#define MCSR0_GR64 (_ULCAST_(1) << MCSR0_GR64_SHIFT) +#define GR64_DEFAULT 1 +#define MCSR0_GR32_SHIFT 32 +#define MCSR0_GR32 (_ULCAST_(1) << MCSR0_GR32_SHIFT) +#define GR32_DEFAULT 0 +#define MCSR0_PRID_WIDTH 32 +#define MCSR0_PRID 0x14C010 + +#define LOONGARCH_CSR_MCSR1 0xc1 /* CPUCFG2 and CPUCFG3 */ +#define MCSR1_HPFOLD_SHIFT 43 +#define MCSR1_HPFOLD (_ULCAST_(1) << MCSR1_HPFOLD_SHIFT) +#define MCSR1_SPW_LVL_SHIFT 40 +#define MCSR1_SPW_LVL_WIDTH 3 +#define MCSR1_SPW_LVL (_ULCAST_(7) << MCSR1_SPW_LVL_SHIFT) +#define MCSR1_ICACHET_SHIFT 39 +#define MCSR1_ICACHET (_ULCAST_(1) << MCSR1_ICACHET_SHIFT) +#define MCSR1_ITLBT_SHIFT 38 +#define MCSR1_ITLBT (_ULCAST_(1) << MCSR1_ITLBT_SHIFT) +#define MCSR1_LLDBAR_SHIFT 37 +#define MCSR1_LLDBAR (_ULCAST_(1) << MCSR1_LLDBAR_SHIFT) +#define MCSR1_SCDLY_SHIFT 36 +#define MCSR1_SCDLY (_ULCAST_(1) << MCSR1_SCDLY_SHIFT) +#define MCSR1_LLEXC_SHIFT 35 +#define MCSR1_LLEXC (_ULCAST_(1) << MCSR1_LLEXC_SHIFT) +#define MCSR1_UCACC_SHIFT 34 +#define MCSR1_UCACC (_ULCAST_(1) << MCSR1_UCACC_SHIFT) +#define MCSR1_SFB_SHIFT 33 +#define MCSR1_SFB (_ULCAST_(1) << MCSR1_SFB_SHIFT) +#define MCSR1_CCDMA_SHIFT 32 +#define MCSR1_CCDMA (_ULCAST_(1) << MCSR1_CCDMA_SHIFT) +#define MCSR1_LAMO_SHIFT 22 +#define MCSR1_LAMO (_ULCAST_(1) << MCSR1_LAMO_SHIFT) +#define MCSR1_LSPW_SHIFT 21 +#define MCSR1_LSPW (_ULCAST_(1) << MCSR1_LSPW_SHIFT) +#define MCSR1_MIPSBT_SHIFT 20 +#define MCSR1_MIPSBT (_ULCAST_(1) << MCSR1_MIPSBT_SHIFT) +#define MCSR1_ARMBT_SHIFT 19 +#define MCSR1_ARMBT (_ULCAST_(1) << MCSR1_ARMBT_SHIFT) +#define MCSR1_X86BT_SHIFT 18 +#define MCSR1_X86BT (_ULCAST_(1) << MCSR1_X86BT_SHIFT) +#define MCSR1_LLFTPVERS_SHIFT 15 +#define MCSR1_LLFTPVERS_WIDTH 3 +#define MCSR1_LLFTPVERS (_ULCAST_(7) << MCSR1_LLFTPVERS_SHIFT) +#define MCSR1_LLFTP_SHIFT 14 +#define MCSR1_LLFTP (_ULCAST_(1) << MCSR1_LLFTP_SHIFT) +#define MCSR1_VZVERS_SHIFT 11 +#define MCSR1_VZVERS_WIDTH 3 +#define MCSR1_VZVERS (_ULCAST_(7) << MCSR1_VZVERS_SHIFT) +#define MCSR1_VZ_SHIFT 10 +#define MCSR1_VZ (_ULCAST_(1) << MCSR1_VZ_SHIFT) +#define MCSR1_CRYPTO_SHIFT 9 +#define MCSR1_CRYPTO (_ULCAST_(1) << MCSR1_CRYPTO_SHIFT) +#define MCSR1_COMPLEX_SHIFT 8 +#define MCSR1_COMPLEX (_ULCAST_(1) << MCSR1_COMPLEX_SHIFT) +#define MCSR1_LASX_SHIFT 7 +#define MCSR1_LASX (_ULCAST_(1) << MCSR1_LASX_SHIFT) +#define MCSR1_LSX_SHIFT 6 +#define MCSR1_LSX (_ULCAST_(1) << MCSR1_LSX_SHIFT) +#define MCSR1_FPVERS_SHIFT 3 +#define MCSR1_FPVERS_WIDTH 3 +#define MCSR1_FPVERS (_ULCAST_(7) << MCSR1_FPVERS_SHIFT) +#define MCSR1_FPDP_SHIFT 2 +#define MCSR1_FPDP (_ULCAST_(1) << MCSR1_FPDP_SHIFT) +#define MCSR1_FPSP_SHIFT 1 +#define MCSR1_FPSP (_ULCAST_(1) << MCSR1_FPSP_SHIFT) +#define MCSR1_FP_SHIFT 0 +#define MCSR1_FP (_ULCAST_(1) << MCSR1_FP_SHIFT) + +#define LOONGARCH_CSR_MCSR2 0xc2 /* CPUCFG4 and CPUCFG5 */ +#define MCSR2_CCDIV_SHIFT 48 +#define MCSR2_CCDIV_WIDTH 16 +#define MCSR2_CCDIV (_ULCAST_(0xffff) << MCSR2_CCDIV_SHIFT) +#define MCSR2_CCMUL_SHIFT 32 +#define MCSR2_CCMUL_WIDTH 16 +#define MCSR2_CCMUL (_ULCAST_(0xffff) << MCSR2_CCMUL_SHIFT) +#define MCSR2_CCFREQ_WIDTH 32 +#define MCSR2_CCFREQ (_ULCAST_(0xffffffff)) +#define CCFREQ_DEFAULT 0x5f5e100 /* 100MHz */ + +#define LOONGARCH_CSR_MCSR3 0xc3 /* CPUCFG6 */ +#define MCSR3_UPM_SHIFT 14 +#define MCSR3_UPM (_ULCAST_(1) << MCSR3_UPM_SHIFT) +#define MCSR3_PMBITS_SHIFT 8 +#define MCSR3_PMBITS_WIDTH 6 +#define MCSR3_PMBITS (_ULCAST_(0x3f) << MCSR3_PMBITS_SHIFT) +#define PMBITS_DEFAULT 0x40 +#define MCSR3_PMNUM_SHIFT 4 +#define MCSR3_PMNUM_WIDTH 4 +#define MCSR3_PMNUM (_ULCAST_(0xf) << MCSR3_PMNUM_SHIFT) +#define MCSR3_PAMVER_SHIFT 1 +#define MCSR3_PAMVER_WIDTH 3 +#define MCSR3_PAMVER (_ULCAST_(0x7) << MCSR3_PAMVER_SHIFT) +#define MCSR3_PMP_SHIFT 0 +#define MCSR3_PMP (_ULCAST_(1) << MCSR3_PMP_SHIFT) + +#define LOONGARCH_CSR_MCSR8 0xc8 /* CPUCFG16 and CPUCFG17 */ +#define MCSR8_L1I_SIZE_SHIFT 56 +#define MCSR8_L1I_SIZE_WIDTH 7 +#define MCSR8_L1I_SIZE (_ULCAST_(0x7f) << MCSR8_L1I_SIZE_SHIFT) +#define MCSR8_L1I_IDX_SHIFT 48 +#define MCSR8_L1I_IDX_WIDTH 8 +#define MCSR8_L1I_IDX (_ULCAST_(0xff) << MCSR8_L1I_IDX_SHIFT) +#define MCSR8_L1I_WAY_SHIFT 32 +#define MCSR8_L1I_WAY_WIDTH 16 +#define MCSR8_L1I_WAY (_ULCAST_(0xffff) << MCSR8_L1I_WAY_SHIFT) +#define MCSR8_L3DINCL_SHIFT 16 +#define MCSR8_L3DINCL (_ULCAST_(1) << MCSR8_L3DINCL_SHIFT) +#define MCSR8_L3DPRIV_SHIFT 15 +#define MCSR8_L3DPRIV (_ULCAST_(1) << MCSR8_L3DPRIV_SHIFT) +#define MCSR8_L3DPRE_SHIFT 14 +#define MCSR8_L3DPRE (_ULCAST_(1) << MCSR8_L3DPRE_SHIFT) +#define MCSR8_L3IUINCL_SHIFT 13 +#define MCSR8_L3IUINCL (_ULCAST_(1) << MCSR8_L3IUINCL_SHIFT) +#define MCSR8_L3IUPRIV_SHIFT 12 +#define MCSR8_L3IUPRIV (_ULCAST_(1) << MCSR8_L3IUPRIV_SHIFT) +#define MCSR8_L3IUUNIFY_SHIFT 11 +#define MCSR8_L3IUUNIFY (_ULCAST_(1) << MCSR8_L3IUUNIFY_SHIFT) +#define MCSR8_L3IUPRE_SHIFT 10 +#define MCSR8_L3IUPRE (_ULCAST_(1) << MCSR8_L3IUPRE_SHIFT) +#define MCSR8_L2DINCL_SHIFT 9 +#define MCSR8_L2DINCL (_ULCAST_(1) << MCSR8_L2DINCL_SHIFT) +#define MCSR8_L2DPRIV_SHIFT 8 +#define MCSR8_L2DPRIV (_ULCAST_(1) << MCSR8_L2DPRIV_SHIFT) +#define MCSR8_L2DPRE_SHIFT 7 +#define MCSR8_L2DPRE (_ULCAST_(1) << MCSR8_L2DPRE_SHIFT) +#define MCSR8_L2IUINCL_SHIFT 6 +#define MCSR8_L2IUINCL (_ULCAST_(1) << MCSR8_L2IUINCL_SHIFT) +#define MCSR8_L2IUPRIV_SHIFT 5 +#define MCSR8_L2IUPRIV (_ULCAST_(1) << MCSR8_L2IUPRIV_SHIFT) +#define MCSR8_L2IUUNIFY_SHIFT 4 +#define MCSR8_L2IUUNIFY (_ULCAST_(1) << MCSR8_L2IUUNIFY_SHIFT) +#define MCSR8_L2IUPRE_SHIFT 3 +#define MCSR8_L2IUPRE (_ULCAST_(1) << MCSR8_L2IUPRE_SHIFT) +#define MCSR8_L1DPRE_SHIFT 2 +#define MCSR8_L1DPRE (_ULCAST_(1) << MCSR8_L1DPRE_SHIFT) +#define MCSR8_L1IUUNIFY_SHIFT 1 +#define MCSR8_L1IUUNIFY (_ULCAST_(1) << MCSR8_L1IUUNIFY_SHIFT) +#define MCSR8_L1IUPRE_SHIFT 0 +#define MCSR8_L1IUPRE (_ULCAST_(1) << MCSR8_L1IUPRE_SHIFT) + +#define LOONGARCH_CSR_MCSR9 0xc9 /* CPUCFG18 and CPUCFG19 */ +#define MCSR9_L2U_SIZE_SHIFT 56 +#define MCSR9_L2U_SIZE_WIDTH 7 +#define MCSR9_L2U_SIZE (_ULCAST_(0x7f) << MCSR9_L2U_SIZE_SHIFT) +#define MCSR9_L2U_IDX_SHIFT 48 +#define MCSR9_L2U_IDX_WIDTH 8 +#define MCSR9_L2U_IDX (_ULCAST_(0xff) << MCSR9_IDX_LOG_SHIFT) +#define MCSR9_L2U_WAY_SHIFT 32 +#define MCSR9_L2U_WAY_WIDTH 16 +#define MCSR9_L2U_WAY (_ULCAST_(0xffff) << MCSR9_L2U_WAY_SHIFT) +#define MCSR9_L1D_SIZE_SHIFT 24 +#define MCSR9_L1D_SIZE_WIDTH 7 +#define MCSR9_L1D_SIZE (_ULCAST_(0x7f) << MCSR9_L1D_SIZE_SHIFT) +#define MCSR9_L1D_IDX_SHIFT 16 +#define MCSR9_L1D_IDX_WIDTH 8 +#define MCSR9_L1D_IDX (_ULCAST_(0xff) << MCSR9_L1D_IDX_SHIFT) +#define MCSR9_L1D_WAY_SHIFT 0 +#define MCSR9_L1D_WAY_WIDTH 16 +#define MCSR9_L1D_WAY (_ULCAST_(0xffff) << MCSR9_L1D_WAY_SHIFT) + +#define LOONGARCH_CSR_MCSR10 0xca /* CPUCFG20 */ +#define MCSR10_L3U_SIZE_SHIFT 24 +#define MCSR10_L3U_SIZE_WIDTH 7 +#define MCSR10_L3U_SIZE (_ULCAST_(0x7f) << MCSR10_L3U_SIZE_SHIFT) +#define MCSR10_L3U_IDX_SHIFT 16 +#define MCSR10_L3U_IDX_WIDTH 8 +#define MCSR10_L3U_IDX (_ULCAST_(0xff) << MCSR10_L3U_IDX_SHIFT) +#define MCSR10_L3U_WAY_SHIFT 0 +#define MCSR10_L3U_WAY_WIDTH 16 +#define MCSR10_L3U_WAY (_ULCAST_(0xffff) << MCSR10_L3U_WAY_SHIFT) + +#define LOONGARCH_CSR_MCSR24 0xf0 /* cpucfg48 */ +#define MCSR24_RAMCG_SHIFT 3 +#define MCSR24_RAMCG (_ULCAST_(1) << MCSR24_RAMCG_SHIFT) +#define MCSR24_VFPUCG_SHIFT 2 +#define MCSR24_VFPUCG (_ULCAST_(1) << MCSR24_VFPUCG_SHIFT) +#define MCSR24_NAPEN_SHIFT 1 +#define MCSR24_NAPEN (_ULCAST_(1) << MCSR24_NAPEN_SHIFT) +#define MCSR24_MCSRLOCK_SHIFT 0 +#define MCSR24_MCSRLOCK (_ULCAST_(1) << MCSR24_MCSRLOCK_SHIFT) + +/* Uncached accelerate windows registers */ +#define LOONGARCH_CSR_UCAWIN 0x100 +#define LOONGARCH_CSR_UCAWIN0_LO 0x102 +#define LOONGARCH_CSR_UCAWIN0_HI 0x103 +#define LOONGARCH_CSR_UCAWIN1_LO 0x104 +#define LOONGARCH_CSR_UCAWIN1_HI 0x105 +#define LOONGARCH_CSR_UCAWIN2_LO 0x106 +#define LOONGARCH_CSR_UCAWIN2_HI 0x107 +#define LOONGARCH_CSR_UCAWIN3_LO 0x108 +#define LOONGARCH_CSR_UCAWIN3_HI 0x109 + +/* Direct Map windows registers */ +#define LOONGARCH_CSR_DMWIN0 0x180 /* 64 direct map win0: MEM & IF */ +#define LOONGARCH_CSR_DMWIN1 0x181 /* 64 direct map win1: MEM & IF */ +#define LOONGARCH_CSR_DMWIN2 0x182 /* 64 direct map win2: MEM */ +#define LOONGARCH_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ + +/* Direct Map window 0/1 */ +#define CSR_DMW0_PLV0 _CONST64_(1 << 0) +#define CSR_DMW0_VSEG _CONST64_(0x8000) +#define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) +#define CSR_DMW0_INIT (CSR_DMW0_BASE | CSR_DMW0_PLV0) + +#define CSR_DMW1_PLV0 _CONST64_(1 << 0) +#define CSR_DMW1_MAT _CONST64_(1 << 4) +#define CSR_DMW1_VSEG _CONST64_(0x9000) +#define CSR_DMW1_BASE (CSR_DMW1_VSEG << DMW_PABITS) +#define CSR_DMW1_INIT (CSR_DMW1_BASE | CSR_DMW1_MAT | CSR_DMW1_PLV0) + +/* Performance Counter registers */ +#define LOONGARCH_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ +#define LOONGARCH_CSR_PERFCNTR0 0x201 /* 64 perf event 0 count value */ +#define LOONGARCH_CSR_PERFCTRL1 0x202 /* 32 perf event 1 config */ +#define LOONGARCH_CSR_PERFCNTR1 0x203 /* 64 perf event 1 count value */ +#define LOONGARCH_CSR_PERFCTRL2 0x204 /* 32 perf event 2 config */ +#define LOONGARCH_CSR_PERFCNTR2 0x205 /* 64 perf event 2 count value */ +#define LOONGARCH_CSR_PERFCTRL3 0x206 /* 32 perf event 3 config */ +#define LOONGARCH_CSR_PERFCNTR3 0x207 /* 64 perf event 3 count value */ +#define CSR_PERFCTRL_PLV0 (_ULCAST_(1) << 16) +#define CSR_PERFCTRL_PLV1 (_ULCAST_(1) << 17) +#define CSR_PERFCTRL_PLV2 (_ULCAST_(1) << 18) +#define CSR_PERFCTRL_PLV3 (_ULCAST_(1) << 19) +#define CSR_PERFCTRL_IE (_ULCAST_(1) << 20) +#define CSR_PERFCTRL_GMOD (_ULCAST_(3) << 21) +#define CSR_PERFCTRL_EVENT 0x3ff + +/* Debug registers */ +#define LOONGARCH_CSR_MWPC 0x300 /* data breakpoint config */ +#define LOONGARCH_CSR_MWPS 0x301 /* data breakpoint status */ + +#define LOONGARCH_CSR_DB0ADDR 0x310 /* data breakpoint 0 address */ +#define LOONGARCH_CSR_DB0MASK 0x311 /* data breakpoint 0 mask */ +#define LOONGARCH_CSR_DB0CTL 0x312 /* data breakpoint 0 control */ +#define LOONGARCH_CSR_DB0ASID 0x313 /* data breakpoint 0 asid */ + +#define LOONGARCH_CSR_DB1ADDR 0x318 /* data breakpoint 1 address */ +#define LOONGARCH_CSR_DB1MASK 0x319 /* data breakpoint 1 mask */ +#define LOONGARCH_CSR_DB1CTL 0x31a /* data breakpoint 1 control */ +#define LOONGARCH_CSR_DB1ASID 0x31b /* data breakpoint 1 asid */ + +#define LOONGARCH_CSR_DB2ADDR 0x320 /* data breakpoint 2 address */ +#define LOONGARCH_CSR_DB2MASK 0x321 /* data breakpoint 2 mask */ +#define LOONGARCH_CSR_DB2CTL 0x322 /* data breakpoint 2 control */ +#define LOONGARCH_CSR_DB2ASID 0x323 /* data breakpoint 2 asid */ + +#define LOONGARCH_CSR_DB3ADDR 0x328 /* data breakpoint 3 address */ +#define LOONGARCH_CSR_DB3MASK 0x329 /* data breakpoint 3 mask */ +#define LOONGARCH_CSR_DB3CTL 0x32a /* data breakpoint 3 control */ +#define LOONGARCH_CSR_DB3ASID 0x32b /* data breakpoint 3 asid */ + +#define LOONGARCH_CSR_DB4ADDR 0x330 /* data breakpoint 4 address */ +#define LOONGARCH_CSR_DB4MASK 0x331 /* data breakpoint 4 maks */ +#define LOONGARCH_CSR_DB4CTL 0x332 /* data breakpoint 4 control */ +#define LOONGARCH_CSR_DB4ASID 0x333 /* data breakpoint 4 asid */ + +#define LOONGARCH_CSR_DB5ADDR 0x338 /* data breakpoint 5 address */ +#define LOONGARCH_CSR_DB5MASK 0x339 /* data breakpoint 5 mask */ +#define LOONGARCH_CSR_DB5CTL 0x33a /* data breakpoint 5 control */ +#define LOONGARCH_CSR_DB5ASID 0x33b /* data breakpoint 5 asid */ + +#define LOONGARCH_CSR_DB6ADDR 0x340 /* data breakpoint 6 address */ +#define LOONGARCH_CSR_DB6MASK 0x341 /* data breakpoint 6 mask */ +#define LOONGARCH_CSR_DB6CTL 0x342 /* data breakpoint 6 control */ +#define LOONGARCH_CSR_DB6ASID 0x343 /* data breakpoint 6 asid */ + +#define LOONGARCH_CSR_DB7ADDR 0x348 /* data breakpoint 7 address */ +#define LOONGARCH_CSR_DB7MASK 0x349 /* data breakpoint 7 mask */ +#define LOONGARCH_CSR_DB7CTL 0x34a /* data breakpoint 7 control */ +#define LOONGARCH_CSR_DB7ASID 0x34b /* data breakpoint 7 asid */ + +#define LOONGARCH_CSR_FWPC 0x380 /* instruction breakpoint config */ +#define LOONGARCH_CSR_FWPS 0x381 /* instruction breakpoint status */ + +#define LOONGARCH_CSR_IB0ADDR 0x390 /* inst breakpoint 0 address */ +#define LOONGARCH_CSR_IB0MASK 0x391 /* inst breakpoint 0 mask */ +#define LOONGARCH_CSR_IB0CTL 0x392 /* inst breakpoint 0 control */ +#define LOONGARCH_CSR_IB0ASID 0x393 /* inst breakpoint 0 asid */ + +#define LOONGARCH_CSR_IB1ADDR 0x398 /* inst breakpoint 1 address */ +#define LOONGARCH_CSR_IB1MASK 0x399 /* inst breakpoint 1 mask */ +#define LOONGARCH_CSR_IB1CTL 0x39a /* inst breakpoint 1 control */ +#define LOONGARCH_CSR_IB1ASID 0x39b /* inst breakpoint 1 asid */ + +#define LOONGARCH_CSR_IB2ADDR 0x3a0 /* inst breakpoint 2 address */ +#define LOONGARCH_CSR_IB2MASK 0x3a1 /* inst breakpoint 2 mask */ +#define LOONGARCH_CSR_IB2CTL 0x3a2 /* inst breakpoint 2 control */ +#define LOONGARCH_CSR_IB2ASID 0x3a3 /* inst breakpoint 2 asid */ + +#define LOONGARCH_CSR_IB3ADDR 0x3a8 /* inst breakpoint 3 address */ +#define LOONGARCH_CSR_IB3MASK 0x3a9 /* breakpoint 3 mask */ +#define LOONGARCH_CSR_IB3CTL 0x3aa /* inst breakpoint 3 control */ +#define LOONGARCH_CSR_IB3ASID 0x3ab /* inst breakpoint 3 asid */ + +#define LOONGARCH_CSR_IB4ADDR 0x3b0 /* inst breakpoint 4 address */ +#define LOONGARCH_CSR_IB4MASK 0x3b1 /* inst breakpoint 4 mask */ +#define LOONGARCH_CSR_IB4CTL 0x3b2 /* inst breakpoint 4 control */ +#define LOONGARCH_CSR_IB4ASID 0x3b3 /* inst breakpoint 4 asid */ + +#define LOONGARCH_CSR_IB5ADDR 0x3b8 /* inst breakpoint 5 address */ +#define LOONGARCH_CSR_IB5MASK 0x3b9 /* inst breakpoint 5 mask */ +#define LOONGARCH_CSR_IB5CTL 0x3ba /* inst breakpoint 5 control */ +#define LOONGARCH_CSR_IB5ASID 0x3bb /* inst breakpoint 5 asid */ + +#define LOONGARCH_CSR_IB6ADDR 0x3c0 /* inst breakpoint 6 address */ +#define LOONGARCH_CSR_IB6MASK 0x3c1 /* inst breakpoint 6 mask */ +#define LOONGARCH_CSR_IB6CTL 0x3c2 /* inst breakpoint 6 control */ +#define LOONGARCH_CSR_IB6ASID 0x3c3 /* inst breakpoint 6 asid */ + +#define LOONGARCH_CSR_IB7ADDR 0x3c8 /* inst breakpoint 7 address */ +#define LOONGARCH_CSR_IB7MASK 0x3c9 /* inst breakpoint 7 mask */ +#define LOONGARCH_CSR_IB7CTL 0x3ca /* inst breakpoint 7 control */ +#define LOONGARCH_CSR_IB7ASID 0x3cb /* inst breakpoint 7 asid */ + +#define LOONGARCH_CSR_DEBUG 0x500 /* debug config */ +#define LOONGARCH_CSR_DERA 0x501 /* debug era */ +#define LOONGARCH_CSR_DESAVE 0x502 /* debug save */ + +/* + * CSR_ECFG IM + */ +#define ECFG0_IM 0x00001fff +#define ECFGB_SIP0 0 +#define ECFGF_SIP0 (_ULCAST_(1) << ECFGB_SIP0) +#define ECFGB_SIP1 1 +#define ECFGF_SIP1 (_ULCAST_(1) << ECFGB_SIP1) +#define ECFGB_IP0 2 +#define ECFGF_IP0 (_ULCAST_(1) << ECFGB_IP0) +#define ECFGB_IP1 3 +#define ECFGF_IP1 (_ULCAST_(1) << ECFGB_IP1) +#define ECFGB_IP2 4 +#define ECFGF_IP2 (_ULCAST_(1) << ECFGB_IP2) +#define ECFGB_IP3 5 +#define ECFGF_IP3 (_ULCAST_(1) << ECFGB_IP3) +#define ECFGB_IP4 6 +#define ECFGF_IP4 (_ULCAST_(1) << ECFGB_IP4) +#define ECFGB_IP5 7 +#define ECFGF_IP5 (_ULCAST_(1) << ECFGB_IP5) +#define ECFGB_IP6 8 +#define ECFGF_IP6 (_ULCAST_(1) << ECFGB_IP6) +#define ECFGB_IP7 9 +#define ECFGF_IP7 (_ULCAST_(1) << ECFGB_IP7) +#define ECFGB_PMC 10 +#define ECFGF_PMC (_ULCAST_(1) << ECFGB_PMC) +#define ECFGB_TIMER 11 +#define ECFGF_TIMER (_ULCAST_(1) << ECFGB_TIMER) +#define ECFGB_IPI 12 +#define ECFGF_IPI (_ULCAST_(1) << ECFGB_IPI) +#define ECFGF(hwirq) (_ULCAST_(1) << hwirq) + +#define ESTATF_IP 0x00003fff + +#define LOONGARCH_IOCSR_FEATURES 0x8 +#define IOCSRF_TEMP BIT_ULL(0) +#define IOCSRF_NODECNT BIT_ULL(1) +#define IOCSRF_MSI BIT_ULL(2) +#define IOCSRF_EXTIOI BIT_ULL(3) +#define IOCSRF_CSRIPI BIT_ULL(4) +#define IOCSRF_FREQCSR BIT_ULL(5) +#define IOCSRF_FREQSCALE BIT_ULL(6) +#define IOCSRF_DVFSV1 BIT_ULL(7) +#define IOCSRF_EIODECODE BIT_ULL(9) +#define IOCSRF_FLATMODE BIT_ULL(10) +#define IOCSRF_VM BIT_ULL(11) + +#define LOONGARCH_IOCSR_VENDOR 0x10 + +#define LOONGARCH_IOCSR_CPUNAME 0x20 + +#define LOONGARCH_IOCSR_NODECNT 0x408 + +#define LOONGARCH_IOCSR_MISC_FUNC 0x420 +#define IOCSR_MISC_FUNC_SOFT_INT BIT_ULL(10) +#define IOCSR_MISC_FUNC_TIMER_RESET BIT_ULL(21) +#define IOCSR_MISC_FUNC_EXT_IOI_EN BIT_ULL(48) + +#define LOONGARCH_IOCSR_CPUTEMP 0x428 + +#define LOONGARCH_IOCSR_SMCMBX 0x51c + +/* PerCore CSR, only accessable by local cores */ +#define LOONGARCH_IOCSR_IPI_STATUS 0x1000 +#define LOONGARCH_IOCSR_IPI_EN 0x1004 +#define LOONGARCH_IOCSR_IPI_SET 0x1008 +#define LOONGARCH_IOCSR_IPI_CLEAR 0x100c +#define LOONGARCH_IOCSR_MBUF0 0x1020 +#define LOONGARCH_IOCSR_MBUF1 0x1028 +#define LOONGARCH_IOCSR_MBUF2 0x1030 +#define LOONGARCH_IOCSR_MBUF3 0x1038 + +#define LOONGARCH_IOCSR_IPI_SEND 0x1040 +#define IOCSR_IPI_SEND_IP_SHIFT 0 +#define IOCSR_IPI_SEND_CPU_SHIFT 16 +#define IOCSR_IPI_SEND_BLOCKING BIT(31) + +#define LOONGARCH_IOCSR_MBUF_SEND 0x1048 +#define IOCSR_MBUF_SEND_BLOCKING BIT_ULL(31) +#define IOCSR_MBUF_SEND_BOX_SHIFT 2 +#define IOCSR_MBUF_SEND_BOX_LO(box) (box << 1) +#define IOCSR_MBUF_SEND_BOX_HI(box) ((box << 1) + 1) +#define IOCSR_MBUF_SEND_CPU_SHIFT 16 +#define IOCSR_MBUF_SEND_BUF_SHIFT 32 +#define IOCSR_MBUF_SEND_H32_MASK 0xFFFFFFFF00000000ULL + +#define LOONGARCH_IOCSR_ANY_SEND 0x1158 +#define IOCSR_ANY_SEND_BLOCKING BIT_ULL(31) +#define IOCSR_ANY_SEND_CPU_SHIFT 16 +#define IOCSR_ANY_SEND_MASK_SHIFT 27 +#define IOCSR_ANY_SEND_BUF_SHIFT 32 +#define IOCSR_ANY_SEND_H32_MASK 0xFFFFFFFF00000000ULL + +/* Register offset and bit definition for CSR access */ +#define LOONGARCH_IOCSR_TIMER_CFG 0x1060 +#define LOONGARCH_IOCSR_TIMER_TICK 0x1070 +#define IOCSR_TIMER_CFG_RESERVED (_ULCAST_(1) << 63) +#define IOCSR_TIMER_CFG_PERIODIC (_ULCAST_(1) << 62) +#define IOCSR_TIMER_CFG_EN (_ULCAST_(1) << 61) +#define IOCSR_TIMER_MASK 0x0ffffffffffffULL +#define IOCSR_TIMER_INITVAL_RST (_ULCAST_(0xffff) << 48) + +#define LOONGARCH_IOCSR_EXTIOI_NODEMAP_BASE 0x14a0 +#define LOONGARCH_IOCSR_EXTIOI_IPMAP_BASE 0x14c0 +#define LOONGARCH_IOCSR_EXTIOI_EN_BASE 0x1600 +#define LOONGARCH_IOCSR_EXTIOI_BOUNCE_BASE 0x1680 +#define LOONGARCH_IOCSR_EXTIOI_ISR_BASE 0x1800 +#define LOONGARCH_IOCSR_EXTIOI_ROUTE_BASE 0x1c00 +#define IOCSR_EXTIOI_VECTOR_NUM 256 + +#ifndef __ASSEMBLY__ + +static inline u64 drdtime(void) +{ + u64 val = 0; + + __asm__ __volatile__( + "rdtime.d %0, $zero\n\t" + : "=r"(val) + : + ); + return val; +} + +static inline unsigned int get_csr_cpuid(void) +{ + return csr_read32(LOONGARCH_CSR_CPUID); +} + +static inline void csr_any_send(unsigned int addr, unsigned int data, + unsigned int data_mask, unsigned int cpu) +{ + uint64_t val = 0; + + val = IOCSR_ANY_SEND_BLOCKING | addr; + val |= (cpu << IOCSR_ANY_SEND_CPU_SHIFT); + val |= (data_mask << IOCSR_ANY_SEND_MASK_SHIFT); + val |= ((uint64_t)data << IOCSR_ANY_SEND_BUF_SHIFT); + iocsr_write64(val, LOONGARCH_IOCSR_ANY_SEND); +} + +static inline unsigned int read_csr_excode(void) +{ + return (csr_read32(LOONGARCH_CSR_ESTAT) & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; +} + +static inline void write_csr_index(unsigned int idx) +{ + csr_xchg32(idx, CSR_TLBIDX_IDXM, LOONGARCH_CSR_TLBIDX); +} + +static inline unsigned int read_csr_pagesize(void) +{ + return (csr_read32(LOONGARCH_CSR_TLBIDX) & CSR_TLBIDX_SIZEM) >> CSR_TLBIDX_SIZE; +} + +static inline void write_csr_pagesize(unsigned int size) +{ + csr_xchg32(size << CSR_TLBIDX_SIZE, CSR_TLBIDX_SIZEM, LOONGARCH_CSR_TLBIDX); +} + +static inline unsigned int read_csr_tlbrefill_pagesize(void) +{ + return (csr_read64(LOONGARCH_CSR_TLBREHI) & CSR_TLBREHI_PS) >> CSR_TLBREHI_PS_SHIFT; +} + +static inline void write_csr_tlbrefill_pagesize(unsigned int size) +{ + csr_xchg64(size << CSR_TLBREHI_PS_SHIFT, CSR_TLBREHI_PS, LOONGARCH_CSR_TLBREHI); +} + +#define read_csr_asid() csr_read32(LOONGARCH_CSR_ASID) +#define write_csr_asid(val) csr_write32(val, LOONGARCH_CSR_ASID) +#define read_csr_entryhi() csr_read64(LOONGARCH_CSR_TLBEHI) +#define write_csr_entryhi(val) csr_write64(val, LOONGARCH_CSR_TLBEHI) +#define read_csr_entrylo0() csr_read64(LOONGARCH_CSR_TLBELO0) +#define write_csr_entrylo0(val) csr_write64(val, LOONGARCH_CSR_TLBELO0) +#define read_csr_entrylo1() csr_read64(LOONGARCH_CSR_TLBELO1) +#define write_csr_entrylo1(val) csr_write64(val, LOONGARCH_CSR_TLBELO1) +#define read_csr_ecfg() csr_read32(LOONGARCH_CSR_ECFG) +#define write_csr_ecfg(val) csr_write32(val, LOONGARCH_CSR_ECFG) +#define read_csr_estat() csr_read32(LOONGARCH_CSR_ESTAT) +#define write_csr_estat(val) csr_write32(val, LOONGARCH_CSR_ESTAT) +#define read_csr_tlbidx() csr_read32(LOONGARCH_CSR_TLBIDX) +#define write_csr_tlbidx(val) csr_write32(val, LOONGARCH_CSR_TLBIDX) +#define read_csr_euen() csr_read32(LOONGARCH_CSR_EUEN) +#define write_csr_euen(val) csr_write32(val, LOONGARCH_CSR_EUEN) +#define read_csr_cpuid() csr_read32(LOONGARCH_CSR_CPUID) +#define read_csr_prcfg1() csr_read64(LOONGARCH_CSR_PRCFG1) +#define write_csr_prcfg1(val) csr_write64(val, LOONGARCH_CSR_PRCFG1) +#define read_csr_prcfg2() csr_read64(LOONGARCH_CSR_PRCFG2) +#define write_csr_prcfg2(val) csr_write64(val, LOONGARCH_CSR_PRCFG2) +#define read_csr_prcfg3() csr_read64(LOONGARCH_CSR_PRCFG3) +#define write_csr_prcfg3(val) csr_write64(val, LOONGARCH_CSR_PRCFG3) +#define read_csr_stlbpgsize() csr_read32(LOONGARCH_CSR_STLBPGSIZE) +#define write_csr_stlbpgsize(val) csr_write32(val, LOONGARCH_CSR_STLBPGSIZE) +#define read_csr_rvacfg() csr_read32(LOONGARCH_CSR_RVACFG) +#define write_csr_rvacfg(val) csr_write32(val, LOONGARCH_CSR_RVACFG) +#define write_csr_tintclear(val) csr_write32(val, LOONGARCH_CSR_TINTCLR) +#define read_csr_impctl1() csr_read64(LOONGARCH_CSR_IMPCTL1) +#define write_csr_impctl1(val) csr_write64(val, LOONGARCH_CSR_IMPCTL1) +#define write_csr_impctl2(val) csr_write64(val, LOONGARCH_CSR_IMPCTL2) + +#define read_csr_perfctrl0() csr_read64(LOONGARCH_CSR_PERFCTRL0) +#define read_csr_perfcntr0() csr_read64(LOONGARCH_CSR_PERFCNTR0) +#define read_csr_perfctrl1() csr_read64(LOONGARCH_CSR_PERFCTRL1) +#define read_csr_perfcntr1() csr_read64(LOONGARCH_CSR_PERFCNTR1) +#define read_csr_perfctrl2() csr_read64(LOONGARCH_CSR_PERFCTRL2) +#define read_csr_perfcntr2() csr_read64(LOONGARCH_CSR_PERFCNTR2) +#define read_csr_perfctrl3() csr_read64(LOONGARCH_CSR_PERFCTRL3) +#define read_csr_perfcntr3() csr_read64(LOONGARCH_CSR_PERFCNTR3) +#define write_csr_perfctrl0(val) csr_write64(val, LOONGARCH_CSR_PERFCTRL0) +#define write_csr_perfcntr0(val) csr_write64(val, LOONGARCH_CSR_PERFCNTR0) +#define write_csr_perfctrl1(val) csr_write64(val, LOONGARCH_CSR_PERFCTRL1) +#define write_csr_perfcntr1(val) csr_write64(val, LOONGARCH_CSR_PERFCNTR1) +#define write_csr_perfctrl2(val) csr_write64(val, LOONGARCH_CSR_PERFCTRL2) +#define write_csr_perfcntr2(val) csr_write64(val, LOONGARCH_CSR_PERFCNTR2) +#define write_csr_perfctrl3(val) csr_write64(val, LOONGARCH_CSR_PERFCTRL3) +#define write_csr_perfcntr3(val) csr_write64(val, LOONGARCH_CSR_PERFCNTR3) + +/* Guest related CSRS */ +#define read_csr_gtlbc() csr_read32(LOONGARCH_CSR_GTLBC) +#define write_csr_gtlbc(val) csr_write32(val, LOONGARCH_CSR_GTLBC) +#define read_csr_trgp() csr_read32(LOONGARCH_CSR_TRGP) +#define read_csr_gcfg() csr_read32(LOONGARCH_CSR_GCFG) +#define write_csr_gcfg(val) csr_write32(val, LOONGARCH_CSR_GCFG) +#define read_csr_gstat() csr_read32(LOONGARCH_CSR_GSTAT) +#define write_csr_gstat(val) csr_write32(val, LOONGARCH_CSR_GSTAT) +#define read_csr_gintc() csr_read32(LOONGARCH_CSR_GINTC) +#define write_csr_gintc(val) csr_write32(val, LOONGARCH_CSR_GINTC) +#define read_csr_gcntc() csr_read64(LOONGARCH_CSR_GCNTC) +#define write_csr_gcntc(val) csr_write64(val, LOONGARCH_CSR_GCNTC) + +/* Guest CSRS read and write */ +#define read_gcsr_crmd() gcsr_read(LOONGARCH_CSR_CRMD) +#define write_gcsr_crmd(val) gcsr_write(val, LOONGARCH_CSR_CRMD) +#define read_gcsr_prmd() gcsr_read(LOONGARCH_CSR_PRMD) +#define write_gcsr_prmd(val) gcsr_write(val, LOONGARCH_CSR_PRMD) +#define read_gcsr_euen() gcsr_read(LOONGARCH_CSR_EUEN) +#define write_gcsr_euen(val) gcsr_write(val, LOONGARCH_CSR_EUEN) +#define read_gcsr_misc() gcsr_read(LOONGARCH_CSR_MISC) +#define write_gcsr_misc(val) gcsr_write(val, LOONGARCH_CSR_MISC) +#define read_gcsr_ecfg() gcsr_read(LOONGARCH_CSR_ECFG) +#define write_gcsr_ecfg(val) gcsr_write(val, LOONGARCH_CSR_ECFG) +#define read_gcsr_estat() gcsr_read(LOONGARCH_CSR_ESTAT) +#define write_gcsr_estat(val) gcsr_write(val, LOONGARCH_CSR_ESTAT) +#define read_gcsr_era() gcsr_read(LOONGARCH_CSR_ERA) +#define write_gcsr_era(val) gcsr_write(val, LOONGARCH_CSR_ERA) +#define read_gcsr_badv() gcsr_read(LOONGARCH_CSR_BADV) +#define write_gcsr_badv(val) gcsr_write(val, LOONGARCH_CSR_BADV) +#define read_gcsr_badi() gcsr_read(LOONGARCH_CSR_BADI) +#define write_gcsr_badi(val) gcsr_write(val, LOONGARCH_CSR_BADI) +#define read_gcsr_eentry() gcsr_read(LOONGARCH_CSR_EENTRY) +#define write_gcsr_eentry(val) gcsr_write(val, LOONGARCH_CSR_EENTRY) + +#define read_gcsr_tlbidx() gcsr_read(LOONGARCH_CSR_TLBIDX) +#define write_gcsr_tlbidx(val) gcsr_write(val, LOONGARCH_CSR_TLBIDX) +#define read_gcsr_tlbhi() gcsr_read(LOONGARCH_CSR_TLBEHI) +#define write_gcsr_tlbhi(val) gcsr_write(val, LOONGARCH_CSR_TLBEHI) +#define read_gcsr_tlblo0() gcsr_read(LOONGARCH_CSR_TLBELO0) +#define write_gcsr_tlblo0(val) gcsr_write(val, LOONGARCH_CSR_TLBELO0) +#define read_gcsr_tlblo1() gcsr_read(LOONGARCH_CSR_TLBELO1) +#define write_gcsr_tlblo1(val) gcsr_write(val, LOONGARCH_CSR_TLBELO1) + +#define read_gcsr_asid() gcsr_read(LOONGARCH_CSR_ASID) +#define write_gcsr_asid(val) gcsr_write(val, LOONGARCH_CSR_ASID) +#define read_gcsr_pgdl() gcsr_read(LOONGARCH_CSR_PGDL) +#define write_gcsr_pgdl(val) gcsr_write(val, LOONGARCH_CSR_PGDL) +#define read_gcsr_pgdh() gcsr_read(LOONGARCH_CSR_PGDH) +#define write_gcsr_pgdh(val) gcsr_write(val, LOONGARCH_CSR_PGDH) +#define read_gcsr_pgd() gcsr_read(LOONGARCH_CSR_PGD) +#define write_gcsr_pgd(val) gcsr_write(val, LOONGARCH_CSR_PGD) +#define read_gcsr_pwctl0() gcsr_read(LOONGARCH_CSR_PWCTL0) +#define write_gcsr_pwctl0(val) gcsr_write(val, LOONGARCH_CSR_PWCTL0) +#define read_gcsr_pwctl1() gcsr_read(LOONGARCH_CSR_PWCTL1) +#define write_gcsr_pwctl1(val) gcsr_write(val, LOONGARCH_CSR_PWCTL1) +#define read_gcsr_stlbpgsize() gcsr_read(LOONGARCH_CSR_STLBPGSIZE) +#define write_gcsr_stlbpgsize(val) gcsr_write(val, LOONGARCH_CSR_STLBPGSIZE) +#define read_gcsr_rvacfg() gcsr_read(LOONGARCH_CSR_RVACFG) +#define write_gcsr_rvacfg(val) gcsr_write(val, LOONGARCH_CSR_RVACFG) + +#define read_gcsr_cpuid() gcsr_read(LOONGARCH_CSR_CPUID) +#define write_gcsr_cpuid(val) gcsr_write(val, LOONGARCH_CSR_CPUID) +#define read_gcsr_prcfg1() gcsr_read(LOONGARCH_CSR_PRCFG1) +#define write_gcsr_prcfg1(val) gcsr_write(val, LOONGARCH_CSR_PRCFG1) +#define read_gcsr_prcfg2() gcsr_read(LOONGARCH_CSR_PRCFG2) +#define write_gcsr_prcfg2(val) gcsr_write(val, LOONGARCH_CSR_PRCFG2) +#define read_gcsr_prcfg3() gcsr_read(LOONGARCH_CSR_PRCFG3) +#define write_gcsr_prcfg3(val) gcsr_write(val, LOONGARCH_CSR_PRCFG3) + +#define read_gcsr_kscratch0() gcsr_read(LOONGARCH_CSR_KS0) +#define write_gcsr_kscratch0(val) gcsr_write(val, LOONGARCH_CSR_KS0) +#define read_gcsr_kscratch1() gcsr_read(LOONGARCH_CSR_KS1) +#define write_gcsr_kscratch1(val) gcsr_write(val, LOONGARCH_CSR_KS1) +#define read_gcsr_kscratch2() gcsr_read(LOONGARCH_CSR_KS2) +#define write_gcsr_kscratch2(val) gcsr_write(val, LOONGARCH_CSR_KS2) +#define read_gcsr_kscratch3() gcsr_read(LOONGARCH_CSR_KS3) +#define write_gcsr_kscratch3(val) gcsr_write(val, LOONGARCH_CSR_KS3) +#define read_gcsr_kscratch4() gcsr_read(LOONGARCH_CSR_KS4) +#define write_gcsr_kscratch4(val) gcsr_write(val, LOONGARCH_CSR_KS4) +#define read_gcsr_kscratch5() gcsr_read(LOONGARCH_CSR_KS5) +#define write_gcsr_kscratch5(val) gcsr_write(val, LOONGARCH_CSR_KS5) +#define read_gcsr_kscratch6() gcsr_read(LOONGARCH_CSR_KS6) +#define write_gcsr_kscratch6(val) gcsr_write(val, LOONGARCH_CSR_KS6) +#define read_gcsr_kscratch7() gcsr_read(LOONGARCH_CSR_KS7) +#define write_gcsr_kscratch7(val) gcsr_write(val, LOONGARCH_CSR_KS7) + +#define read_gcsr_timerid() gcsr_read(LOONGARCH_CSR_TMID) +#define write_gcsr_timerid(val) gcsr_write(val, LOONGARCH_CSR_TMID) +#define read_gcsr_timercfg() gcsr_read(LOONGARCH_CSR_TCFG) +#define write_gcsr_timercfg(val) gcsr_write(val, LOONGARCH_CSR_TCFG) +#define read_gcsr_timertick() gcsr_read(LOONGARCH_CSR_TVAL) +#define write_gcsr_timertick(val) gcsr_write(val, LOONGARCH_CSR_TVAL) +#define read_gcsr_timeroffset() gcsr_read(LOONGARCH_CSR_CNTC) +#define write_gcsr_timeroffset(val) gcsr_write(val, LOONGARCH_CSR_CNTC) + +#define read_gcsr_llbctl() gcsr_read(LOONGARCH_CSR_LLBCTL) +#define write_gcsr_llbctl(val) gcsr_write(val, LOONGARCH_CSR_LLBCTL) + +#define read_gcsr_tlbrentry() gcsr_read(LOONGARCH_CSR_TLBRENTRY) +#define write_gcsr_tlbrentry(val) gcsr_write(val, LOONGARCH_CSR_TLBRENTRY) +#define read_gcsr_tlbrbadv() gcsr_read(LOONGARCH_CSR_TLBRBADV) +#define write_gcsr_tlbrbadv(val) gcsr_write(val, LOONGARCH_CSR_TLBRBADV) +#define read_gcsr_tlbrera() gcsr_read(LOONGARCH_CSR_TLBRERA) +#define write_gcsr_tlbrera(val) gcsr_write(val, LOONGARCH_CSR_TLBRERA) +#define read_gcsr_tlbrsave() gcsr_read(LOONGARCH_CSR_TLBRSAVE) +#define write_gcsr_tlbrsave(val) gcsr_write(val, LOONGARCH_CSR_TLBRSAVE) +#define read_gcsr_tlbrelo0() gcsr_read(LOONGARCH_CSR_TLBRELO0) +#define write_gcsr_tlbrelo0(val) gcsr_write(val, LOONGARCH_CSR_TLBRELO0) +#define read_gcsr_tlbrelo1() gcsr_read(LOONGARCH_CSR_TLBRELO1) +#define write_gcsr_tlbrelo1(val) gcsr_write(val, LOONGARCH_CSR_TLBRELO1) +#define read_gcsr_tlbrehi() gcsr_read(LOONGARCH_CSR_TLBREHI) +#define write_gcsr_tlbrehi(val) gcsr_write(val, LOONGARCH_CSR_TLBREHI) +#define read_gcsr_tlbrprmd() gcsr_read(LOONGARCH_CSR_TLBRPRMD) +#define write_gcsr_tlbrprmd(val) gcsr_write(val, LOONGARCH_CSR_TLBRPRMD) + +#define read_gcsr_directwin0() gcsr_read(LOONGARCH_CSR_DMWIN0) +#define write_gcsr_directwin0(val) gcsr_write(val, LOONGARCH_CSR_DMWIN0) +#define read_gcsr_directwin1() gcsr_read(LOONGARCH_CSR_DMWIN1) +#define write_gcsr_directwin1(val) gcsr_write(val, LOONGARCH_CSR_DMWIN1) +#define read_gcsr_directwin2() gcsr_read(LOONGARCH_CSR_DMWIN2) +#define write_gcsr_directwin2(val) gcsr_write(val, LOONGARCH_CSR_DMWIN2) +#define read_gcsr_directwin3() gcsr_read(LOONGARCH_CSR_DMWIN3) +#define write_gcsr_directwin3(val) gcsr_write(val, LOONGARCH_CSR_DMWIN3) + +/* + * Manipulate bits in a register. + */ +#define __BUILD_CSR_COMMON(name) \ +static inline unsigned long \ +set_##name(unsigned long set) \ +{ \ + unsigned long res, new; \ + \ + res = read_##name(); \ + new = res | set; \ + write_##name(new); \ + \ + return res; \ +} \ + \ +static inline unsigned long \ +clear_##name(unsigned long clear) \ +{ \ + unsigned long res, new; \ + \ + res = read_##name(); \ + new = res & ~clear; \ + write_##name(new); \ + \ + return res; \ +} \ + \ +static inline unsigned long \ +change_##name(unsigned long change, unsigned long val) \ +{ \ + unsigned long res, new; \ + \ + res = read_##name(); \ + new = res & ~change; \ + new |= (val & change); \ + write_##name(new); \ + \ + return res; \ +} + +#define __BUILD_CSR_OP(name) __BUILD_CSR_COMMON(csr_##name) +#define __BUILD_GCSR_OP(name) __BUILD_CSR_COMMON(gcsr_##name) + +__BUILD_CSR_OP(euen) +__BUILD_CSR_OP(ecfg) +__BUILD_CSR_OP(tlbidx) +__BUILD_CSR_OP(gcfg) +__BUILD_CSR_OP(gstat) +__BUILD_CSR_OP(gtlbc) +__BUILD_CSR_OP(gintc) +__BUILD_GCSR_OP(llbctl) +__BUILD_GCSR_OP(tlbidx) + +#define set_csr_estat(val) \ + csr_xchg32(val, val, LOONGARCH_CSR_ESTAT) +#define clear_csr_estat(val) \ + csr_xchg32(~(val), val, LOONGARCH_CSR_ESTAT) +#define set_gcsr_estat(val) \ + gcsr_xchg(val, val, LOONGARCH_CSR_ESTAT) +#define clear_gcsr_estat(val) \ + gcsr_xchg(~(val), val, LOONGARCH_CSR_ESTAT) + +#endif /* __ASSEMBLY__ */ + +/* Generic EntryLo bit definitions */ +#define ENTRYLO_V (_ULCAST_(1) << 0) +#define ENTRYLO_D (_ULCAST_(1) << 1) +#define ENTRYLO_PLV_SHIFT 2 +#define ENTRYLO_PLV (_ULCAST_(3) << ENTRYLO_PLV_SHIFT) +#define ENTRYLO_C_SHIFT 4 +#define ENTRYLO_C (_ULCAST_(3) << ENTRYLO_C_SHIFT) +#define ENTRYLO_G (_ULCAST_(1) << 6) +#define ENTRYLO_NR (_ULCAST_(1) << 61) +#define ENTRYLO_NX (_ULCAST_(1) << 62) + +/* Values for PageMask register */ +#define PS_4K 0x0000000c +#define PS_8K 0x0000000d +#define PS_16K 0x0000000e +#define PS_32K 0x0000000f +#define PS_64K 0x00000010 +#define PS_128K 0x00000011 +#define PS_256K 0x00000012 +#define PS_512K 0x00000013 +#define PS_1M 0x00000014 +#define PS_2M 0x00000015 +#define PS_4M 0x00000016 +#define PS_8M 0x00000017 +#define PS_16M 0x00000018 +#define PS_32M 0x00000019 +#define PS_64M 0x0000001a +#define PS_128M 0x0000001b +#define PS_256M 0x0000001c +#define PS_512M 0x0000001d +#define PS_1G 0x0000001e + +/* Default page size for a given kernel configuration */ +#ifdef CONFIG_PAGE_SIZE_4KB +#define PS_DEFAULT_SIZE PS_4K +#elif defined(CONFIG_PAGE_SIZE_16KB) +#define PS_DEFAULT_SIZE PS_16K +#elif defined(CONFIG_PAGE_SIZE_64KB) +#define PS_DEFAULT_SIZE PS_64K +#else +#error Bad page size configuration! +#endif + +/* Default huge tlb size for a given kernel configuration */ +#ifdef CONFIG_PAGE_SIZE_4KB +#define PS_HUGE_SIZE PS_1M +#elif defined(CONFIG_PAGE_SIZE_16KB) +#define PS_HUGE_SIZE PS_16M +#elif defined(CONFIG_PAGE_SIZE_64KB) +#define PS_HUGE_SIZE PS_256M +#else +#error Bad page size configuration for hugetlbfs! +#endif + +/* ExStatus.ExcCode */ +#define EXCCODE_RSV 0 /* Reserved */ +#define EXCCODE_TLBL 1 /* TLB miss on a load */ +#define EXCCODE_TLBS 2 /* TLB miss on a store */ +#define EXCCODE_TLBI 3 /* TLB miss on a ifetch */ +#define EXCCODE_TLBM 4 /* TLB modified fault */ +#define EXCCODE_TLBNR 5 /* TLB Read-Inhibit exception */ +#define EXCCODE_TLBNX 6 /* TLB Execution-Inhibit exception */ +#define EXCCODE_TLBPE 7 /* TLB Privilege Error */ +#define EXCCODE_ADE 8 /* Address Error */ + #define EXSUBCODE_ADEF 0 /* Fetch Instruction */ + #define EXSUBCODE_ADEM 1 /* Access Memory*/ +#define EXCCODE_ALE 9 /* Unalign Access */ +#define EXCCODE_BCE 10 /* Bounds Check Error */ +#define EXCCODE_SYS 11 /* System call */ +#define EXCCODE_BP 12 /* Breakpoint */ +#define EXCCODE_INE 13 /* Inst. Not Exist */ +#define EXCCODE_IPE 14 /* Inst. Privileged Error */ +#define EXCCODE_FPDIS 15 /* FPU Disabled */ +#define EXCCODE_LSXDIS 16 /* LSX Disabled */ +#define EXCCODE_LASXDIS 17 /* LASX Disabled */ +#define EXCCODE_FPE 18 /* Floating Point Exception */ + #define EXCSUBCODE_FPE 0 /* Floating Point Exception */ + #define EXCSUBCODE_VFPE 1 /* Vector Exception */ +#define EXCCODE_WATCH 19 /* WatchPoint Exception */ + #define EXCSUBCODE_WPEF 0 /* ... on Instruction Fetch */ + #define EXCSUBCODE_WPEM 1 /* ... on Memory Accesses */ +#define EXCCODE_BTDIS 20 /* Binary Trans. Disabled */ +#define EXCCODE_BTE 21 /* Binary Trans. Exception */ +#define EXCCODE_GSPR 22 /* Guest Privileged Error */ +#define EXCCODE_HVC 23 /* Hypercall */ +#define EXCCODE_GCM 24 /* Guest CSR modified */ + #define EXCSUBCODE_GCSC 0 /* Software caused */ + #define EXCSUBCODE_GCHC 1 /* Hardware caused */ +#define EXCCODE_SE 25 /* Security */ + +/* Interrupt numbers */ +#define INT_SWI0 0 /* Software Interrupts */ +#define INT_SWI1 1 +#define INT_HWI0 2 /* Hardware Interrupts */ +#define INT_HWI1 3 +#define INT_HWI2 4 +#define INT_HWI3 5 +#define INT_HWI4 6 +#define INT_HWI5 7 +#define INT_HWI6 8 +#define INT_HWI7 9 +#define INT_PCOV 10 /* Performance Counter Overflow */ +#define INT_TI 11 /* Timer */ +#define INT_IPI 12 +#define INT_NMI 13 + +/* ExcCodes corresponding to interrupts */ +#define EXCCODE_INT_NUM (INT_NMI + 1) +#define EXCCODE_INT_START 64 +#define EXCCODE_INT_END (EXCCODE_INT_START + EXCCODE_INT_NUM - 1) + +/* FPU register names */ +#define LOONGARCH_FCSR0 $r0 +#define LOONGARCH_FCSR1 $r1 +#define LOONGARCH_FCSR2 $r2 +#define LOONGARCH_FCSR3 $r3 + +/* FPU Status Register Values */ +#define FPU_CSR_RSVD 0xe0e0fce0 + +/* + * X the exception cause indicator + * E the exception enable + * S the sticky/flag bit + */ +#define FPU_CSR_ALL_X 0x1f000000 +#define FPU_CSR_INV_X 0x10000000 +#define FPU_CSR_DIV_X 0x08000000 +#define FPU_CSR_OVF_X 0x04000000 +#define FPU_CSR_UDF_X 0x02000000 +#define FPU_CSR_INE_X 0x01000000 + +#define FPU_CSR_ALL_S 0x001f0000 +#define FPU_CSR_INV_S 0x00100000 +#define FPU_CSR_DIV_S 0x00080000 +#define FPU_CSR_OVF_S 0x00040000 +#define FPU_CSR_UDF_S 0x00020000 +#define FPU_CSR_INE_S 0x00010000 + +#define FPU_CSR_ALL_E 0x0000001f +#define FPU_CSR_INV_E 0x00000010 +#define FPU_CSR_DIV_E 0x00000008 +#define FPU_CSR_OVF_E 0x00000004 +#define FPU_CSR_UDF_E 0x00000002 +#define FPU_CSR_INE_E 0x00000001 + +/* Bits 8 and 9 of FPU Status Register specify the rounding mode */ +#define FPU_CSR_RM 0x300 +#define FPU_CSR_RN 0x000 /* nearest */ +#define FPU_CSR_RZ 0x100 /* towards zero */ +#define FPU_CSR_RU 0x200 /* towards +Infinity */ +#define FPU_CSR_RD 0x300 /* towards -Infinity */ + +/* LBT extension */ +#define FPU_CSR_TM_SHIFT 0x6 +#define FPU_CSR_TM 0x40 /* float register in stack mode */ + +#define write_fcsr(dest, val) \ +do { \ + __asm__ __volatile__( \ + " movgr2fcsr "__stringify(dest)", %0 \n" \ + : : "r" (val)); \ +} while (0) + +#define read_fcsr(source) \ +({ \ + unsigned int __res; \ +\ + __asm__ __volatile__( \ + " movfcsr2gr %0, "__stringify(source)" \n" \ + : "=r" (__res)); \ + __res; \ +}) + +#endif /* _ASM_LOONGARCHREGS_H */ diff --git a/arch/loongarch/include/asm/mach-loongson64/irq.h b/arch/loongarch/include/asm/mach-loongson64/irq.h new file mode 100644 index 0000000000000000000000000000000000000000..e549199eb20b3aba904480ef12d26ae70cfcf14e --- /dev/null +++ b/arch/loongarch/include/asm/mach-loongson64/irq.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MACH_LOONGSON64_IRQ_H_ +#define __ASM_MACH_LOONGSON64_IRQ_H_ + +#include + +#define MAX_IO_PICS 2 +#define NR_IRQS (64 + (256 * MAX_IO_PICS)) + +#define CORES_PER_EIO_NODE 4 + +#define LOONGSON_CPU_UART0_VEC 10 /* CPU UART0 */ +#define LOONGSON_CPU_THSENS_VEC 14 /* CPU Thsens */ +#define LOONGSON_CPU_HT0_VEC 16 /* CPU HT0 irq vector base number */ +#define LOONGSON_CPU_HT1_VEC 24 /* CPU HT1 irq vector base number */ + +/* IRQ number definitions */ +#define LOONGSON_LPC_IRQ_BASE 0 +#define LOONGSON_LPC_LAST_IRQ (LOONGSON_LPC_IRQ_BASE + 15) + +#define LOONGSON_CPU_IRQ_BASE 16 +#define LOONGSON_CPU_LAST_IRQ (LOONGSON_CPU_IRQ_BASE + 14) + +#define LOONGSON_PCH_IRQ_BASE 64 +#define LOONGSON_PCH_ACPI_IRQ (LOONGSON_PCH_IRQ_BASE + 47) +#define LOONGSON_PCH_LAST_IRQ (LOONGSON_PCH_IRQ_BASE + 64 - 1) + +#define LOONGSON_MSI_IRQ_BASE (LOONGSON_PCH_IRQ_BASE + 64) +#define LOONGSON_MSI_LAST_IRQ (LOONGSON_PCH_IRQ_BASE + 256 - 1) + +#define GSI_MIN_LPC_IRQ LOONGSON_LPC_IRQ_BASE +#define GSI_MAX_LPC_IRQ (LOONGSON_LPC_IRQ_BASE + 16 - 1) +#define GSI_MIN_CPU_IRQ LOONGSON_CPU_IRQ_BASE +#define GSI_MAX_CPU_IRQ (LOONGSON_CPU_IRQ_BASE + 48 - 1) +#define GSI_MIN_PCH_IRQ LOONGSON_PCH_IRQ_BASE +#define GSI_MAX_PCH_IRQ (LOONGSON_PCH_IRQ_BASE + 256 - 1) + +extern int find_pch_pic(u32 gsi); +extern int eiointc_get_node(int id); + +static inline void eiointc_enable(void) +{ + uint64_t misc; + + misc = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC); + misc |= IOCSR_MISC_FUNC_EXT_IOI_EN; + iocsr_write64(misc, LOONGARCH_IOCSR_MISC_FUNC); +} + +struct acpi_madt_lio_pic; +struct acpi_madt_eio_pic; +struct acpi_madt_ht_pic; +struct acpi_madt_bio_pic; +struct acpi_madt_msi_pic; +struct acpi_madt_lpc_pic; + +struct irq_domain *liointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lio_pic *acpi_liointc); +struct irq_domain *eiointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_eio_pic *acpi_eiointc); + +struct irq_domain *htvec_acpi_init(struct irq_domain *parent, + struct acpi_madt_ht_pic *acpi_htvec); +struct irq_domain *pch_lpc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lpc_pic *acpi_pchlpc); +struct irq_domain *pch_msi_acpi_init(struct irq_domain *parent, + struct acpi_madt_msi_pic *acpi_pchmsi); +struct irq_domain *pch_pic_acpi_init(struct irq_domain *parent, + struct acpi_madt_bio_pic *acpi_pchpic); + +extern struct acpi_madt_lio_pic *acpi_liointc; +extern struct acpi_madt_eio_pic *acpi_eiointc[MAX_IO_PICS]; + +extern struct acpi_madt_ht_pic *acpi_htintc; +extern struct acpi_madt_lpc_pic *acpi_pchlpc; +extern struct acpi_madt_msi_pic *acpi_pchmsi[MAX_IO_PICS]; +extern struct acpi_madt_bio_pic *acpi_pchpic[MAX_IO_PICS]; + +extern struct irq_domain *cpu_domain; +extern struct irq_domain *liointc_domain; +extern struct irq_domain *pch_lpc_domain; +extern struct irq_domain *pch_msi_domain[MAX_IO_PICS]; +extern struct irq_domain *pch_pic_domain[MAX_IO_PICS]; + +extern irqreturn_t loongson3_ipi_interrupt(int irq, void *dev); + +#endif /* __ASM_MACH_LOONGSON64_IRQ_H_ */ diff --git a/arch/loongarch/include/asm/mach-loongson64/loongson.h b/arch/loongarch/include/asm/mach-loongson64/loongson.h new file mode 100644 index 0000000000000000000000000000000000000000..48ae4f03e872f105400848d720876555990e071c --- /dev/null +++ b/arch/loongarch/include/asm/mach-loongson64/loongson.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#ifndef __ASM_MACH_LOONGSON64_LOONGSON_H +#define __ASM_MACH_LOONGSON64_LOONGSON_H + +#include +#include +#include +#include +#include +#include + +extern const struct plat_smp_ops loongson3_smp_ops; + +#define LOONGSON_REG(x) \ + (*(volatile u32 *)((char *)TO_UNCACHE(LOONGSON_REG_BASE) + (x))) + +#define LOONGSON_LIO_BASE 0x18000000 +#define LOONGSON_LIO_SIZE 0x00100000 /* 1M */ +#define LOONGSON_LIO_TOP (LOONGSON_LIO_BASE+LOONGSON_LIO_SIZE-1) + +#define LOONGSON_BOOT_BASE 0x1c000000 +#define LOONGSON_BOOT_SIZE 0x02000000 /* 32M */ +#define LOONGSON_BOOT_TOP (LOONGSON_BOOT_BASE+LOONGSON_BOOT_SIZE-1) + +#define LOONGSON_REG_BASE 0x1fe00000 +#define LOONGSON_REG_SIZE 0x00100000 /* 1M */ +#define LOONGSON_REG_TOP (LOONGSON_REG_BASE+LOONGSON_REG_SIZE-1) + +/* GPIO Regs - r/w */ + +#define LOONGSON_GPIODATA LOONGSON_REG(0x11c) +#define LOONGSON_GPIOIE LOONGSON_REG(0x120) +#define LOONGSON_REG_GPIO_BASE (LOONGSON_REG_BASE + 0x11c) + +#define MAX_PACKAGES 16 + +#define xconf_readl(addr) readl(addr) +#define xconf_readq(addr) readq(addr) + +static inline void xconf_writel(u32 val, volatile void __iomem *addr) +{ + asm volatile ( + " st.w %[v], %[hw], 0 \n" + " ld.b $r0, %[hw], 0 \n" + : + : [hw] "r" (addr), [v] "r" (val) + ); +} + +static inline void xconf_writeq(u64 val64, volatile void __iomem *addr) +{ + asm volatile ( + " st.d %[v], %[hw], 0 \n" + " ld.b $r0, %[hw], 0 \n" + : + : [hw] "r" (addr), [v] "r" (val64) + ); +} + +/* ============== LS7A registers =============== */ +#define LS7A_PCH_REG_BASE 0x10000000UL +/* LPC regs */ +#define LS7A_LPC_REG_BASE (LS7A_PCH_REG_BASE + 0x00002000) +/* CHIPCFG regs */ +#define LS7A_CHIPCFG_REG_BASE (LS7A_PCH_REG_BASE + 0x00010000) +/* MISC reg base */ +#define LS7A_MISC_REG_BASE (LS7A_PCH_REG_BASE + 0x00080000) +/* ACPI regs */ +#define LS7A_ACPI_REG_BASE (LS7A_MISC_REG_BASE + 0x00050000) +/* RTC regs */ +#define LS7A_RTC_REG_BASE (LS7A_MISC_REG_BASE + 0x00050100) + +#define LS7A_DMA_CFG (volatile void *)TO_UNCACHE(LS7A_CHIPCFG_REG_BASE + 0x041c) +#define LS7A_DMA_NODE_SHF 8 +#define LS7A_DMA_NODE_MASK 0x1F00 + +#define LS7A_INT_MASK_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x020) +#define LS7A_INT_EDGE_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x060) +#define LS7A_INT_CLEAR_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x080) +#define LS7A_INT_HTMSI_EN_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x040) +#define LS7A_INT_ROUTE_ENTRY_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x100) +#define LS7A_INT_HTMSI_VEC_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x200) +#define LS7A_INT_STATUS_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x3a0) +#define LS7A_INT_POL_REG (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x3e0) +#define LS7A_LPC_INT_CTL (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x2000) +#define LS7A_LPC_INT_ENA (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x2004) +#define LS7A_LPC_INT_STS (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x2008) +#define LS7A_LPC_INT_CLR (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x200c) +#define LS7A_LPC_INT_POL (volatile void *)TO_UNCACHE(LS7A_PCH_REG_BASE + 0x2010) + +#define LS7A_PMCON_SOC_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x000) +#define LS7A_PMCON_RESUME_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x004) +#define LS7A_PMCON_RTC_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x008) +#define LS7A_PM1_EVT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x00c) +#define LS7A_PM1_ENA_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x010) +#define LS7A_PM1_CNT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x014) +#define LS7A_PM1_TMR_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x018) +#define LS7A_P_CNT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x01c) +#define LS7A_GPE0_STS_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x028) +#define LS7A_GPE0_ENA_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x02c) +#define LS7A_RST_CNT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x030) +#define LS7A_WD_SET_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x034) +#define LS7A_WD_TIMER_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x038) +#define LS7A_THSENS_CNT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x04c) +#define LS7A_GEN_RTC_1_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x050) +#define LS7A_GEN_RTC_2_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x054) +#define LS7A_DPM_CFG_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x400) +#define LS7A_DPM_STS_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x404) +#define LS7A_DPM_CNT_REG (volatile void *)TO_UNCACHE(LS7A_ACPI_REG_BASE + 0x408) + +typedef enum { + ACPI_PCI_HOTPLUG_STATUS = 1 << 1, + ACPI_CPU_HOTPLUG_STATUS = 1 << 2, + ACPI_MEM_HOTPLUG_STATUS = 1 << 3, + ACPI_POWERBUTTON_STATUS = 1 << 8, + ACPI_RTC_WAKE_STATUS = 1 << 10, + ACPI_PCI_WAKE_STATUS = 1 << 14, + ACPI_ANY_WAKE_STATUS = 1 << 15, +} AcpiEventStatusBits; + +#define HT1LO_OFFSET 0xe0000000000UL + +/* PCI Configration Space Base */ +#define MCFG_EXT_PCICFG_BASE 0xefe00000000UL + +/* REG ACCESS*/ +#define ls7a_readb(addr) (*(volatile unsigned char *)TO_UNCACHE(addr)) +#define ls7a_readw(addr) (*(volatile unsigned short *)TO_UNCACHE(addr)) +#define ls7a_readl(addr) (*(volatile unsigned int *)TO_UNCACHE(addr)) +#define ls7a_readq(addr) (*(volatile unsigned long *)TO_UNCACHE(addr)) +#define ls7a_writeb(val, addr) *(volatile unsigned char *)TO_UNCACHE(addr) = (val) +#define ls7a_writew(val, addr) *(volatile unsigned short *)TO_UNCACHE(addr) = (val) +#define ls7a_writel(val, addr) *(volatile unsigned int *)TO_UNCACHE(addr) = (val) +#define ls7a_writeq(val, addr) *(volatile unsigned long *)TO_UNCACHE(addr) = (val) + +void enable_gpe_wakeup(void); +void enable_pci_wakeup(void); + +#endif /* __ASM_MACH_LOONGSON64_LOONGSON_H */ diff --git a/arch/loongarch/include/asm/mach-loongson64/topology.h b/arch/loongarch/include/asm/mach-loongson64/topology.h new file mode 100644 index 0000000000000000000000000000000000000000..5cda49ca74ab2329a1d0a32ac00cf99cae6c9d0e --- /dev/null +++ b/arch/loongarch/include/asm/mach-loongson64/topology.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_MACH_TOPOLOGY_H +#define _ASM_MACH_TOPOLOGY_H + +#ifdef CONFIG_NUMA + +extern cpumask_t cpus_on_node[]; + +#define cpumask_of_node(node) (&cpus_on_node[node]) + +struct pci_bus; +extern int pcibus_to_node(struct pci_bus *); + +#define cpumask_of_pcibus(bus) (cpu_online_mask) + +extern unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES]; + +void numa_set_distance(int from, int to, int distance); + +#define node_distance(from, to) (node_distances[(from)][(to)]) + +#else +#define pcibus_to_node(bus) 0 +#endif + +#endif /* _ASM_MACH_TOPOLOGY_H */ diff --git a/arch/loongarch/include/asm/mmu.h b/arch/loongarch/include/asm/mmu.h new file mode 100644 index 0000000000000000000000000000000000000000..c2753acaf0ea9730528c13a78258505127a9fb65 --- /dev/null +++ b/arch/loongarch/include/asm/mmu.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_MMU_H +#define __ASM_MMU_H + +#include +#include + +typedef struct { + u64 asid[NR_CPUS]; + void *vdso; +} mm_context_t; + +#endif /* __ASM_MMU_H */ diff --git a/arch/loongarch/include/asm/mmu_context.h b/arch/loongarch/include/asm/mmu_context.h new file mode 100644 index 0000000000000000000000000000000000000000..ded6107885d0816d136f667e9290670efe926c04 --- /dev/null +++ b/arch/loongarch/include/asm/mmu_context.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * Switch a MMU context. + */ +#ifndef _ASM_MMU_CONTEXT_H +#define _ASM_MMU_CONTEXT_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * All unused by hardware upper bits will be considered + * as a software asid extension. + */ +static inline u64 asid_version_mask(unsigned int cpu) +{ + return ~(u64)(cpu_asid_mask(&cpu_data[cpu])); +} + +static inline u64 asid_first_version(unsigned int cpu) +{ + return cpu_asid_mask(&cpu_data[cpu]) + 1; +} + +#define cpu_context(cpu, mm) ((mm)->context.asid[cpu]) +#define asid_cache(cpu) (cpu_data[cpu].asid_cache) +#define cpu_asid(cpu, mm) (cpu_context((cpu), (mm)) & cpu_asid_mask(&cpu_data[cpu])) + +static inline int asid_valid(struct mm_struct *mm, unsigned int cpu) +{ + if ((cpu_context(cpu, mm) ^ asid_cache(cpu)) & asid_version_mask(cpu)) + return 0; + + return 1; +} + +static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +{ +} + +/* Normal, classic get_new_mmu_context */ +static inline void +get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) +{ + u64 asid = asid_cache(cpu); + + if (!((++asid) & cpu_asid_mask(&cpu_data[cpu]))) + local_flush_tlb_user(); /* start new asid cycle */ + + cpu_context(cpu, mm) = asid_cache(cpu) = asid; +} + +/* + * Initialize the context related info for a new mm_struct + * instance. + */ +static inline int +init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ + int i; + + for_each_possible_cpu(i) + cpu_context(i, mm) = 0; + + return 0; +} + +static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + unsigned int cpu = smp_processor_id(); + + /* Check if our ASID is of an older version and thus invalid */ + if (!asid_valid(next, cpu)) + get_new_mmu_context(next, cpu); + + write_csr_asid(cpu_asid(cpu, next)); + + if (next != &init_mm) + csr_write64((unsigned long)next->pgd, LOONGARCH_CSR_PGDL); + else + csr_write64((unsigned long)invalid_pg_dir, LOONGARCH_CSR_PGDL); + + /* + * Mark current->active_mm as not "active" anymore. + * We don't want to mislead possible IPI tlb flush routines. + */ + cpumask_set_cpu(cpu, mm_cpumask(next)); +} + +#define switch_mm_irqs_off switch_mm_irqs_off + +static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + unsigned long flags; + + local_irq_save(flags); + switch_mm_irqs_off(prev, next, tsk); + local_irq_restore(flags); +} + +/* + * Destroy context related info for an mm_struct that is about + * to be put to rest. + */ +static inline void destroy_context(struct mm_struct *mm) +{ +} + +#define activate_mm(prev, next) switch_mm(prev, next, current) +#define deactivate_mm(task, mm) do { } while (0) + +/* + * If mm is currently active, we can't really drop it. + * Instead, we will get a new one for it. + */ +static inline void +drop_mmu_context(struct mm_struct *mm, unsigned cpu) +{ + int asid; + unsigned long flags; + + local_irq_save(flags); + + asid = read_csr_asid() & cpu_asid_mask(¤t_cpu_data); + + if (asid == cpu_asid(cpu, mm)) { + if (!current->mm || (current->mm == mm)) { + get_new_mmu_context(mm, cpu); + write_csr_asid(cpu_asid(cpu, mm)); + goto out; + } + } + + /* Will get a new context next time */ + cpu_context(cpu, mm) = 0; + cpumask_clear_cpu(cpu, mm_cpumask(mm)); +out: + local_irq_restore(flags); +} + +#endif /* _ASM_MMU_CONTEXT_H */ diff --git a/arch/loongarch/include/asm/mmzone.h b/arch/loongarch/include/asm/mmzone.h new file mode 100644 index 0000000000000000000000000000000000000000..1be0073189fde45cef15d673cd312fa89bb183df --- /dev/null +++ b/arch/loongarch/include/asm/mmzone.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Written by Huacai Chen (chenhuacai@loongson.cn) + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_MMZONE_H_ +#define _ASM_MMZONE_H_ + +#include +#include + +extern struct pglist_data *node_data[]; + +#define NODE_DATA(nid) (node_data[(nid)]) + +#endif /* _ASM_MMZONE_H_ */ diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h new file mode 100644 index 0000000000000000000000000000000000000000..d9e91add666bd1a04a6e1bdfbcd957fa06db038d --- /dev/null +++ b/arch/loongarch/include/asm/module.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_MODULE_H +#define _ASM_MODULE_H + +#include +#include +#include + +#define RELA_STACK_DEPTH 16 + +struct mod_section { + int shndx; + int num_entries; + int max_entries; +}; + +struct mod_arch_specific { + struct mod_section got; + struct mod_section plt; + struct mod_section plt_idx; +#ifdef CONFIG_UNWINDER_ORC + unsigned int num_orcs; + int *orc_unwind_ip; + struct orc_entry *orc_unwind; +#endif + + /* for CONFIG_DYNAMIC_FTRACE */ + struct plt_entry *ftrace_trampolines; +}; + +struct got_entry { + Elf_Addr symbol_addr; +}; + +struct plt_entry { + u32 inst_addu16id; + u32 inst_lu32id; + u32 inst_lu52id; + u32 inst_jirl; +}; + +struct plt_idx_entry { + Elf_Addr symbol_addr; +}; + +Elf_Addr module_emit_got_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val); +Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val); + +static inline struct got_entry emit_got_entry(Elf_Addr val) +{ + return (struct got_entry) { val }; +} + +static inline struct plt_entry emit_plt_entry(unsigned long val) +{ + u32 addu16id, lu32id, lu52id, jirl; + + addu16id = larch_insn_gen_addu16id(REG_T1, REG_ZERO, ADDR_IMM(val, ADDU16ID)); + lu32id = larch_insn_gen_lu32id(REG_T1, ADDR_IMM(val, LU32ID)); + lu52id = larch_insn_gen_lu52id(REG_T1, REG_T1, ADDR_IMM(val, LU52ID)); + jirl = larch_insn_gen_jirl(0, REG_T1, 0, (val & 0xffff)); + + return (struct plt_entry) { addu16id, lu32id, lu52id, jirl }; +} + +static inline struct plt_idx_entry emit_plt_idx_entry(unsigned long val) +{ + return (struct plt_idx_entry) { val }; +} + +static inline int get_plt_idx(unsigned long val, Elf_Shdr *sechdrs, const struct mod_section *sec) +{ + int i; + struct plt_idx_entry *plt_idx = (struct plt_idx_entry *)sechdrs[sec->shndx].sh_addr; + + for (i = 0; i < sec->num_entries; i++) { + if (plt_idx[i].symbol_addr == val) + return i; + } + + return -1; +} + +static inline struct plt_entry *get_plt_entry(unsigned long val, + Elf_Shdr *sechdrs, + const struct mod_section *sec_plt, + const struct mod_section *sec_plt_idx) +{ + int plt_idx = get_plt_idx(val, sechdrs, sec_plt_idx); + struct plt_entry *plt = (struct plt_entry *)sechdrs[sec_plt->shndx].sh_addr; + + if (plt_idx < 0) + return NULL; + + return plt + plt_idx; +} + +static inline struct got_entry *get_got_entry(Elf_Addr val, + Elf_Shdr *sechdrs, + const struct mod_section *sec) +{ + int i; + struct got_entry *got = (struct got_entry *)sechdrs[sec->shndx].sh_addr; + + for (i = 0; i < sec->num_entries; i++) + if (got[i].symbol_addr == val) + return &got[i]; + return NULL; +} + +#endif /* _ASM_MODULE_H */ diff --git a/arch/loongarch/include/asm/module.lds.h b/arch/loongarch/include/asm/module.lds.h new file mode 100644 index 0000000000000000000000000000000000000000..88554f92e0103de153ece66aa2e8b7d1857db1c8 --- /dev/null +++ b/arch/loongarch/include/asm/module.lds.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ +SECTIONS { + . = ALIGN(4); + .got 0 : { BYTE(0) } + .plt 0 : { BYTE(0) } + .plt.idx 0 : { BYTE(0) } + .ftrace_trampoline 0 : { BYTE(0) } +} diff --git a/arch/loongarch/include/asm/numa.h b/arch/loongarch/include/asm/numa.h new file mode 100644 index 0000000000000000000000000000000000000000..f45f6c09fdc51f88c51edac6aefc3a300d15caf4 --- /dev/null +++ b/arch/loongarch/include/asm/numa.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: lvjianmin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_LOONGARCH_NUMA_H +#define _ASM_LOONGARCH_NUMA_H + +#include + +#define NODE_ADDRSPACE_SHIFT 44 + +#define pa_to_nid(addr) (((addr) & 0xf00000000000) >> NODE_ADDRSPACE_SHIFT) +#define nid_to_addrbase(nid) (_ULCAST_(nid) << NODE_ADDRSPACE_SHIFT) + +#ifdef CONFIG_NUMA + +extern int numa_off; +extern s16 __cpuid_to_node[CONFIG_NR_CPUS]; +extern nodemask_t numa_nodes_parsed __initdata; + +struct numa_memblk { + u64 start; + u64 end; + int nid; +}; + +#define NR_NODE_MEMBLKS (MAX_NUMNODES*2) +struct numa_meminfo { + int nr_blks; + struct numa_memblk blk[NR_NODE_MEMBLKS]; +}; + +extern int __init numa_add_memblk(int nodeid, u64 start, u64 end); + +extern void __init early_numa_add_cpu(int cpuid, s16 node); +extern void numa_add_cpu(unsigned int cpu); +extern void numa_remove_cpu(unsigned int cpu); + +static inline void numa_clear_node(int cpu) +{ +} + +static inline void set_cpuid_to_node(int cpuid, s16 node) +{ + __cpuid_to_node[cpuid] = node; +} + +extern int early_cpu_to_node(int cpu); + +#else + +static inline void early_numa_add_cpu(int cpuid, s16 node) { } +static inline void numa_add_cpu(unsigned int cpu) { } +static inline void numa_remove_cpu(unsigned int cpu) { } + +static inline int early_cpu_to_node(int cpu) +{ + return 0; +} + +#endif /* CONFIG_NUMA */ + +#endif /* _ASM_LOONGARCH_NUMA_H */ diff --git a/arch/loongarch/include/asm/orc_lookup.h b/arch/loongarch/include/asm/orc_lookup.h new file mode 100644 index 0000000000000000000000000000000000000000..f41f1e2f9b67ea8fc6513b7e26e044f96fb189f7 --- /dev/null +++ b/arch/loongarch/include/asm/orc_lookup.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2017 Josh Poimboeuf + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_ORC_LOOKUP_H +#define _ASM_ORC_LOOKUP_H + +/* + * This is a lookup table for speeding up access to the .orc_unwind table. + * Given an input address offset, the corresponding lookup table entry + * specifies a subset of the .orc_unwind table to search. + * + * Each block represents the end of the previous range and the start of the + * next range. An extra block is added to give the last range an end. + * + * The block size should be a power of 2 to avoid a costly 'div' instruction. + * + * A block size of 256 was chosen because it roughly doubles unwinder + * performance while only adding ~5% to the ORC data footprint. + */ +#define LOOKUP_BLOCK_ORDER 8 +#define LOOKUP_BLOCK_SIZE (1 << LOOKUP_BLOCK_ORDER) + +#ifndef LINKER_SCRIPT + +extern unsigned int orc_lookup[]; +extern unsigned int orc_lookup_end[]; + +#define LOOKUP_START_IP ((unsigned long)_stext) +#define LOOKUP_STOP_IP ((unsigned long)_etext) + +#endif /* LINKER_SCRIPT */ + +#endif /* _ASM_ORC_LOOKUP_H */ diff --git a/arch/loongarch/include/asm/orc_types.h b/arch/loongarch/include/asm/orc_types.h new file mode 100644 index 0000000000000000000000000000000000000000..40b2800b0d6f80a63686f8f0ee20b46ca05095ef --- /dev/null +++ b/arch/loongarch/include/asm/orc_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2017 Josh Poimboeuf + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_ORC_TYPES_H +#define _ASM_ORC_TYPES_H + +#include +#include + +/* + * The ORC_REG_* registers are base registers which are used to find other + * registers on the stack. + * + * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the + * address of the previous frame: the caller's SP before it called the current + * function. + * + * ORC_REG_UNDEFINED means the corresponding register's value didn't change in + * the current frame. + * + * The most commonly used base registers are SP and FP -- which the previous SP + * is usually based on -- and PREV_SP and UNDEFINED -- which the previous FP is + * usually based on. + */ +#define ORC_REG_UNDEFINED 0 +#define ORC_REG_PREV_SP 1 +#define ORC_REG_SP 2 +#define ORC_REG_FP 3 +#define ORC_REG_MAX 15 + +#ifndef __ASSEMBLY__ +/* + * This struct is more or less a vastly simplified version of the DWARF Call + * Frame Information standard. It contains only the necessary parts of DWARF + * CFI, simplified for ease of access by the in-kernel unwinder. It tells the + * unwinder how to find the previous SP and FP (and sometimes entry regs) on + * the stack for a given code address. Each instance of the struct corresponds + * to one or more code locations. + */ +struct orc_entry { + signed short sp_offset; + signed short fp_offset; + signed short ra_offset; + unsigned int sp_reg:4; + unsigned int fp_reg:4; + unsigned int ra_reg:4; + unsigned int type:2; + unsigned int end:1; + unsigned int unused:1; +}; + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ORC_TYPES_H */ diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h new file mode 100644 index 0000000000000000000000000000000000000000..5931fb3236275122da6d6498476a3d56d9826dc2 --- /dev/null +++ b/arch/loongarch/include/asm/page.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PAGE_H +#define _ASM_PAGE_H + +#include + +/* + * PAGE_SHIFT determines the page size + */ +#ifdef CONFIG_PAGE_SIZE_4KB +#define PAGE_SHIFT 12 +#endif +#ifdef CONFIG_PAGE_SIZE_16KB +#define PAGE_SHIFT 14 +#endif +#ifdef CONFIG_PAGE_SIZE_64KB +#define PAGE_SHIFT 16 +#endif +#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) +#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +#ifndef __ASSEMBLY__ + +#include +#include + +/* + * It's normally defined only for FLATMEM config but it's + * used in our early mem init code for all memory models. + * So always define it. + */ +#define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET) + +extern void clear_page(void * page); +extern void copy_page(void * to, void * from); + +#define clear_user_page(page, vaddr, pg) clear_page(page) +#define copy_user_page(to, from, vaddr, pg) copy_page(to, from) + +extern unsigned long shm_align_mask; + +struct page; +struct vm_area_struct; +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); + +#define __HAVE_ARCH_COPY_USER_HIGHPAGE + +typedef struct { unsigned long pte; } pte_t; +#define pte_val(x) ((x).pte) +#define __pte(x) ((pte_t) { (x) } ) +typedef struct page *pgtable_t; + +typedef struct { unsigned long pgd; } pgd_t; +#define pgd_val(x) ((x).pgd) +#define __pgd(x) ((pgd_t) { (x) } ) + +/* + * Manipulate page protection bits + */ +typedef struct { unsigned long pgprot; } pgprot_t; +#define pgprot_val(x) ((x).pgprot) +#define __pgprot(x) ((pgprot_t) { (x) } ) +#define pte_pgprot(x) __pgprot(pte_val(x) & ~_PFN_MASK) + +#define ptep_buddy(x) ((pte_t *)((unsigned long)(x) ^ sizeof(pte_t))) + +/* + * __pa()/__va() should be used only during mem init. + */ +#define __pa(x) PHYSADDR(x) +#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET)) + +#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) +#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) + +#ifdef CONFIG_FLATMEM + +static inline int pfn_valid(unsigned long pfn) +{ + /* avoid include hell */ + extern unsigned long max_mapnr; + unsigned long pfn_offset = ARCH_PFN_OFFSET; + + return pfn >= pfn_offset && pfn < max_mapnr; +} + +#endif + +#define virt_to_pfn(kaddr) PFN_DOWN(PHYSADDR(kaddr)) +#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) + +extern int __virt_addr_valid(volatile void *kaddr); +#define virt_addr_valid(kaddr) __virt_addr_valid((volatile void *)(kaddr)) + +#define VM_DATA_DEFAULT_FLAGS \ + (VM_READ | VM_WRITE | \ + ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \ + VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) + +extern unsigned long __kaslr_offset; +static inline unsigned long kaslr_offset(void) +{ + return __kaslr_offset; +} + +#include +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_PAGE_H */ diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h new file mode 100644 index 0000000000000000000000000000000000000000..ee766aec04aeae79e7e49aba8ed915c740442963 --- /dev/null +++ b/arch/loongarch/include/asm/paravirt.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_PARAVIRT_H +#define _ASM_LOONGARCH_PARAVIRT_H +#include + +#ifdef CONFIG_PARAVIRT +static inline bool kvm_para_available(void) +{ + return true; +} +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +struct pv_time_ops { + unsigned long long (*steal_clock)(int cpu); +}; +struct kvm_steal_time { + __u64 steal; + __u32 version; + __u32 flags; + __u8 preempted; + __u8 pad[47]; +}; +extern struct pv_time_ops pv_time_ops; + +bool pv_is_native_spin_unlock(void); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return pv_time_ops.steal_clock(cpu); +} + +static inline bool pv_feature_support(int feature) +{ + return kvm_hypercall1(KVM_HC_FUNC_FEATURE, feature) == KVM_RET_SUC; +} +static inline void pv_notify_host(int feature, unsigned long data) +{ + kvm_hypercall2(KVM_HC_FUNC_NOTIFY, feature, data); +} + +#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) +struct qspinlock; + +struct pv_lock_ops { + void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); + void (*queued_spin_unlock)(struct qspinlock *lock); + void (*wait)(u8 *ptr, u8 val); + void (*kick)(int cpu); + bool (*vcpu_is_preempted)(long cpu); +}; + +extern struct pv_lock_ops pv_lock_ops; + +void __init kvm_spinlock_init(void); + +static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, + u32 val) +{ + pv_lock_ops.queued_spin_lock_slowpath(lock, val); +} + +static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) +{ + pv_lock_ops.queued_spin_unlock(lock); +} + +static __always_inline void pv_wait(u8 *ptr, u8 val) +{ + pv_lock_ops.wait(ptr, val); +} + +static __always_inline void pv_kick(int cpu) +{ + pv_lock_ops.kick(cpu); +} + +static __always_inline bool pv_vcpu_is_preempted(long cpu) +{ + return pv_lock_ops.vcpu_is_preempted(cpu); +} + +#endif /* SMP && PARAVIRT_SPINLOCKS */ + +int __init pv_time_init(void); +int __init pv_ipi_init(void); +#else +static inline bool kvm_para_available(void) +{ + return false; +} + +#define pv_time_init() do {} while (0) +#define pv_ipi_init() do {} while (0) +#endif +#endif /* _ASM_LOONGARCH_PARAVIRT_H */ diff --git a/arch/loongarch/include/asm/pci.h b/arch/loongarch/include/asm/pci.h new file mode 100644 index 0000000000000000000000000000000000000000..6698c01b54f56b01d2add3afbe9488657a9d63bd --- /dev/null +++ b/arch/loongarch/include/asm/pci.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_PCI_H +#define _ASM_PCI_H + +#include + +/* + * This file essentially defines the interface between board specific + * PCI code and LoongArch common PCI code. Should potentially put into + * include/asm/pci.h file. + */ + +#include +#include + +extern phys_addr_t mcfg_addr_init(int node); + +/* Can be used to override the logic in pci_scan_bus for skipping + already-configured bus numbers - to be used for buggy BIOSes + or architectures with incomplete PCI setup by the loader */ +static inline unsigned int pcibios_assign_all_busses(void) +{ + return 0; +} + +#define PCIBIOS_MIN_IO 0x4000 +#define PCIBIOS_MIN_MEM 0x20000000 +#define PCIBIOS_MIN_CARDBUS_IO 0x4000 + +#define HAVE_PCI_MMAP +#define ARCH_GENERIC_PCI_MMAP_RESOURCE + +/* + * Dynamic DMA mapping stuff. + * LoongArch has everything mapped statically. + */ + +#include +#include +#include +#include +#include + +static inline int pci_proc_domain(struct pci_bus *bus) +{ + return 1; /* always show the domain in /proc */ +} + +/* generic pci stuff */ +#include + +#endif /* _ASM_PCI_H */ diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h new file mode 100644 index 0000000000000000000000000000000000000000..3599b826dca4a6c3f1b42694c1a7bc7b0bd412f1 --- /dev/null +++ b/arch/loongarch/include/asm/percpu.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Ltd. + */ +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +#include + +/* + * The "address" (in fact, offset from $r21) of a per-CPU variable is close to + * the loading address of main kernel image, but far from where the modules are + * loaded. Tell the compiler this fact when using explicit relocs. + */ +#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) +# if __has_attribute(model) +# define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) +# else +# error compiler support for the model attribute is necessary when a recent assembler is used +# endif +#endif + +/* Use r21 for fast access */ +register unsigned long __my_cpu_offset __asm__("$r21"); + +static inline void set_my_cpu_offset(unsigned long off) +{ + __my_cpu_offset = off; + csr_write64(off, PERCPU_BASE_KS); +} +#define __my_cpu_offset __my_cpu_offset + +#define PERCPU_OP(op, asm_op, c_op) \ +static __always_inline unsigned long __percpu_##op(void *ptr, \ + unsigned long val, int size) \ +{ \ + unsigned long ret; \ + \ + switch (size) { \ + case 4: \ + __asm__ __volatile__( \ + "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \ + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \ + : [val] "r" (val)); \ + break; \ + case 8: \ + __asm__ __volatile__( \ + "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \ + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \ + : [val] "r" (val)); \ + break; \ + default: \ + ret = 0; \ + BUILD_BUG(); \ + } \ + \ + return ret c_op val; \ +} + +PERCPU_OP(add, add, +) +PERCPU_OP(and, and, &) +PERCPU_OP(or, or, |) +#undef PERCPU_OP + +static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size) +{ + unsigned long ret; + + switch (size) { + case 1: + __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" + : [ret] "=&r"(ret) + : [ptr] "r"(ptr) + : "memory"); + break; + case 2: + __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" + : [ret] "=&r"(ret) + : [ptr] "r"(ptr) + : "memory"); + break; + case 4: + __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" + : [ret] "=&r"(ret) + : [ptr] "r"(ptr) + : "memory"); + break; + case 8: + __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" + : [ret] "=&r"(ret) + : [ptr] "r"(ptr) + : "memory"); + break; + default: + ret = 0; + BUILD_BUG(); + } + + return ret; +} + +static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size) +{ + switch (size) { + case 1: + __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" + : + : [val] "r" (val), [ptr] "r" (ptr) + : "memory"); + break; + case 2: + __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" + : + : [val] "r" (val), [ptr] "r" (ptr) + : "memory"); + break; + case 4: + __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" + : + : [val] "r" (val), [ptr] "r" (ptr) + : "memory"); + break; + case 8: + __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" + : + : [val] "r" (val), [ptr] "r" (ptr) + : "memory"); + break; + default: + BUILD_BUG(); + } +} + +static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size) +{ + switch (size) { + case 1: + case 2: + return __xchg_small((volatile void *)ptr, val, size); + + case 4: + return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); + + case 8: + return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val); + + default: + BUILD_BUG(); + } + + return 0; +} + +/* this_cpu_cmpxchg */ +#define _protect_cmpxchg_local(pcp, o, n) \ +({ \ + typeof(*raw_cpu_ptr(&(pcp))) __ret; \ + preempt_disable_notrace(); \ + __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ + preempt_enable_notrace(); \ + __ret; \ +}) + +#define _percpu_read(pcp) \ +({ \ + typeof(pcp) __retval; \ + __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ + __retval; \ +}) + +#define _percpu_write(pcp, val) \ +do { \ + __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ +} while (0) \ + +#define _pcp_protect(operation, pcp, val) \ +({ \ + typeof(pcp) __retval; \ + preempt_disable_notrace(); \ + __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ + (val), sizeof(pcp)); \ + preempt_enable_notrace(); \ + __retval; \ +}) + +#define _percpu_add(pcp, val) \ + _pcp_protect(__percpu_add, pcp, val) + +#define _percpu_add_return(pcp, val) _percpu_add(pcp, val) + +#define _percpu_and(pcp, val) \ + _pcp_protect(__percpu_and, pcp, val) + +#define _percpu_or(pcp, val) \ + _pcp_protect(__percpu_or, pcp, val) + +#define _percpu_xchg(pcp, val) ((typeof(pcp)) \ + _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) + +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) + +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) + +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) + +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) + +#define this_cpu_read_1(pcp) _percpu_read(pcp) +#define this_cpu_read_2(pcp) _percpu_read(pcp) +#define this_cpu_read_4(pcp) _percpu_read(pcp) +#define this_cpu_read_8(pcp) _percpu_read(pcp) + +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) + +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) + +#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) + +#include + +#endif /* __ASM_PERCPU_H */ diff --git a/arch/loongarch/include/asm/perf_event.h b/arch/loongarch/include/asm/perf_event.h new file mode 100644 index 0000000000000000000000000000000000000000..a9caaf15738db3a5bfe05c070fe20bff0e7fbf22 --- /dev/null +++ b/arch/loongarch/include/asm/perf_event.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/arch/loongarch/include/asm/perf_event.h + * + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __LOONGARCH_PERF_EVENT_H__ +#define __LOONGARCH_PERF_EVENT_H__ + +#define perf_arch_bpf_user_pt_regs(regs) (struct user_pt_regs *)regs + +#endif /* __LOONGARCH_PERF_EVENT_H__ */ diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h new file mode 100644 index 0000000000000000000000000000000000000000..146a61e6f4a1a591ddd3c127b0cc4c4b8c3eee7b --- /dev/null +++ b/arch/loongarch/include/asm/pgalloc.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PGALLOC_H +#define _ASM_PGALLOC_H + +#include +#include + +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#include + +static inline void pmd_populate_kernel(struct mm_struct *mm, + pmd_t *pmd, pte_t *pte) +{ + set_pmd(pmd, __pmd((unsigned long)pte)); +} + +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) +{ + set_pmd(pmd, __pmd((unsigned long)page_address(pte))); +} +#define pmd_pgtable(pmd) pmd_page(pmd) + +#ifndef __PAGETABLE_PMD_FOLDED + +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + set_pud(pud, __pud((unsigned long)pmd)); +} +#endif + +#ifndef __PAGETABLE_PUD_FOLDED + +static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) +{ + set_p4d(p4d, __p4d((unsigned long)pud)); +} + +#endif /* __PAGETABLE_PUD_FOLDED */ + +extern void pagetable_init(void); + +/* + * Initialize a new pmd table with invalid pointers. + */ +extern void pmd_init(unsigned long page, unsigned long pagetable); + +/* + * Initialize a new pgd / pmd table with invalid pointers. + */ +extern void pgd_init(unsigned long page); +extern pgd_t *pgd_alloc(struct mm_struct *mm); + +#define __pte_free_tlb(tlb,pte,address) \ +do { \ + pgtable_pte_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) + +#ifndef __PAGETABLE_PMD_FOLDED + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) +{ + pmd_t *pmd; + struct page *pg; + + pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_ORDER); + if (!pg) + return NULL; + + if (!pgtable_pmd_page_ctor(pg)) { + __free_pages(pg, PMD_ORDER); + return NULL; + } + + pmd = (pmd_t *)page_address(pg); + pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table); + return pmd; +} + +#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) + +#endif + +#ifndef __PAGETABLE_PUD_FOLDED + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) +{ + pud_t *pud; + + pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER); + if (pud) + pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table); + return pud; +} + +#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) + +#endif /* __PAGETABLE_PUD_FOLDED */ + +extern pte_t * __init populate_kernel_pte(unsigned long addr); +#endif /* _ASM_PGALLOC_H */ diff --git a/arch/loongarch/include/asm/pgtable-64.h b/arch/loongarch/include/asm/pgtable-64.h new file mode 100644 index 0000000000000000000000000000000000000000..0ede81e1147b95509296ec9cb4eb523538dab891 --- /dev/null +++ b/arch/loongarch/include/asm/pgtable-64.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PGTABLE_64_H +#define _ASM_PGTABLE_64_H + +#include +#include + +#include + +#if CONFIG_PGTABLE_LEVELS == 2 +#include +#elif CONFIG_PGTABLE_LEVELS == 3 +#include +#else +#include +#endif + +#define PGD_ORDER 0 +#define PUD_ORDER 0 +#define PMD_ORDER 0 +#define PTE_ORDER 0 + +#if CONFIG_PGTABLE_LEVELS == 2 +#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) +#elif CONFIG_PGTABLE_LEVELS == 3 +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) +#elif CONFIG_PGTABLE_LEVELS == 4 +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) +#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3)) +#endif + +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT + PGD_ORDER - 3)) + +#define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) >> 3) +#if CONFIG_PGTABLE_LEVELS > 3 +#define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) >> 3) +#endif +#if CONFIG_PGTABLE_LEVELS > 2 +#define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) >> 3) +#endif +#define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) >> 3) + +#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) +#define FIRST_USER_ADDRESS 0UL + +#ifndef __ASSEMBLY__ + +#include +#include + +/* + * TLB refill handlers may also map the vmalloc area into xkvrange. + * Avoid the first couple of pages so NULL pointer dereferences will + * still reliably trap. + */ +#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) +#define MODULES_END (MODULES_VADDR + SZ_256M) + +#define VMALLOC_START MODULES_END + +#ifndef CONFIG_KASAN +#define VMALLOC_END \ + (vm_map_base + \ + min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE - VMEMMAP_SIZE) +#else +#define VMALLOC_END \ + (vm_map_base + \ + min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits) / 2) - PMD_SIZE - VMEMMAP_SIZE) +#endif + +#define vmemmap ((struct page *)((VMALLOC_END + PMD_SIZE) & PMD_MASK)) +#define VMEMMAP_END ((unsigned long)vmemmap + VMEMMAP_SIZE - 1) + +#define pte_ERROR(e) \ + printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) +#ifndef __PAGETABLE_PMD_FOLDED +#define pmd_ERROR(e) \ + printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) +#endif +#ifndef __PAGETABLE_PUD_FOLDED +#define pud_ERROR(e) \ + printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) +#endif +#define pgd_ERROR(e) \ + printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) + +extern pte_t invalid_pte_table[PTRS_PER_PTE]; + +#ifndef __PAGETABLE_PUD_FOLDED + +typedef struct { unsigned long pud; } pud_t; +#define pud_val(x) ((x).pud) +#define __pud(x) ((pud_t) { (x) }) + +extern pud_t invalid_pud_table[PTRS_PER_PUD]; + +/* + * Empty pgd/p4d entries point to the invalid_pud_table. + */ +static inline int p4d_none(p4d_t p4d) +{ + return p4d_val(p4d) == (unsigned long)invalid_pud_table; +} + +static inline int p4d_bad(p4d_t p4d) +{ + return p4d_val(p4d) & ~PAGE_MASK; +} + +static inline int p4d_present(p4d_t p4d) +{ + return p4d_val(p4d) != (unsigned long)invalid_pud_table; +} + +static inline void p4d_clear(p4d_t *p4dp) +{ + p4d_val(*p4dp) = (unsigned long)invalid_pud_table; +} + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *)p4d_val(p4d); +} + +static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) +{ + *p4d = p4dval; +} + +#define p4d_phys(p4d) PHYSADDR(p4d_val(p4d)) +#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) + +#endif + +#ifndef __PAGETABLE_PMD_FOLDED + +typedef struct { unsigned long pmd; } pmd_t; +#define pmd_val(x) ((x).pmd) +#define __pmd(x) ((pmd_t) { (x) } ) + +extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; + +/* + * Empty pud entries point to the invalid_pmd_table. + */ +static inline int pud_none(pud_t pud) +{ + return pud_val(pud) == (unsigned long)invalid_pmd_table; +} + +static inline int pud_bad(pud_t pud) +{ + return pud_val(pud) & ~PAGE_MASK; +} + +static inline int pud_present(pud_t pud) +{ + return pud_val(pud) != (unsigned long)invalid_pmd_table; +} + +static inline void pud_clear(pud_t *pudp) +{ + pud_val(*pudp) = ((unsigned long)invalid_pmd_table); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)pud_val(pud); +} + +#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0) + +#define pud_phys(pud) PHYSADDR(pud_val(pud)) +#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) + +#endif + +/* + * Empty pmd entries point to the invalid_pte_table. + */ +static inline int pmd_none(pmd_t pmd) +{ + return pmd_val(pmd) == (unsigned long)invalid_pte_table; +} + +static inline int pmd_bad(pmd_t pmd) +{ + return (pmd_val(pmd) & ~PAGE_MASK); +} + +static inline int pmd_present(pmd_t pmd) +{ + if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) + return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PRESENT_INVALID)); + + return pmd_val(pmd) != (unsigned long)invalid_pte_table; +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + pmd_val(*pmdp) = ((unsigned long)invalid_pte_table); +} + +#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0) + +#define pmd_phys(pmd) PHYSADDR(pmd_val(pmd)) + +#ifndef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define pmd_page_vaddr(pmd) pmd_val(pmd) + +extern pmd_t mk_pmd(struct page *page, pgprot_t prot); +extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd); + +#define pte_page(x) pfn_to_page(pte_pfn(x)) +#define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> _PFN_SHIFT)) +#define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) +#define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) + +/* + * Initialize a new pgd / pmd table with invalid pointers. + */ +extern void pgd_init(unsigned long page); +extern void pud_init(unsigned long page, unsigned long pagetable); +extern void pmd_init(unsigned long page, unsigned long pagetable); + +/* + * Non-present pages: high 40 bits are offset, next 8 bits type, + * low 16 bits zero. + */ +static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) +{ pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; } + +#define __swp_type(x) (((x).val >> 16) & 0xff) +#define __swp_offset(x) ((x).val >> 24) +#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) +#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) +#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE }) + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_PGTABLE_64_H */ diff --git a/arch/loongarch/include/asm/pgtable-bits.h b/arch/loongarch/include/asm/pgtable-bits.h new file mode 100644 index 0000000000000000000000000000000000000000..4c980be4065aac0cfaaaea14baabad8cd5e2931e --- /dev/null +++ b/arch/loongarch/include/asm/pgtable-bits.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PGTABLE_BITS_H +#define _ASM_PGTABLE_BITS_H + +/* Page table bits */ +#define _PAGE_VALID_SHIFT 0 +#define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ +#define _PAGE_DIRTY_SHIFT 1 +#define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */ +#define _CACHE_SHIFT 4 /* 4~5, two bits */ +#define _PAGE_GLOBAL_SHIFT 6 +#define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */ +#define _PAGE_PRESENT_SHIFT 7 +#define _PAGE_WRITE_SHIFT 8 +#define _PAGE_MODIFIED_SHIFT 9 +#define _PAGE_PROTNONE_SHIFT 10 +#define _PAGE_SPECIAL_SHIFT 11 +#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ +#define _PAGE_PFN_SHIFT 12 +#define _PAGE_PFN_END_SHIFT 48 +#define _PAGE_PRESENT_INVALID_SHIFT 60 +#define _PAGE_NO_READ_SHIFT 61 +#define _PAGE_NO_EXEC_SHIFT 62 +#define _PAGE_RPLV_SHIFT 63 + +/* Used by software */ +#define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT) +#define _PAGE_PRESENT_INVALID (_ULCAST_(1) << _PAGE_PRESENT_INVALID_SHIFT) +#define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT) +#define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT) +#define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT) +#define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT) +#define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT) + +/* Used by TLB hardware (placed in EntryLo*) */ +#define _PAGE_VALID (_ULCAST_(1) << _PAGE_VALID_SHIFT) +#define _PAGE_DIRTY (_ULCAST_(1) << _PAGE_DIRTY_SHIFT) +#define _PAGE_PLV (_ULCAST_(3) << _PAGE_PLV_SHIFT) +#define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT) +#define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT) +#define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT) +#define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT) +#define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT) +#define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT) +#define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT) +#define _PFN_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT) + +#define _PAGE_USER (PLV_USER << _PAGE_PLV_SHIFT) +#define _PAGE_KERN (PLV_KERN << _PAGE_PLV_SHIFT) + +#define _PFN_MASK (~((_ULCAST_(1) << (_PFN_SHIFT)) - 1) & \ + ((_ULCAST_(1) << (_PAGE_PFN_END_SHIFT)) - 1)) + +/* + * Cache attributes + */ +#ifndef _CACHE_SUC +#define _CACHE_SUC (0<<_CACHE_SHIFT) /* Strong-ordered UnCached */ +#endif +#ifndef _CACHE_CC +#define _CACHE_CC (1<<_CACHE_SHIFT) /* Coherent Cached */ +#endif +#ifndef _CACHE_WUC +#define _CACHE_WUC (2<<_CACHE_SHIFT) /* Weak-ordered UnCached */ +#endif + +#define __READABLE (_PAGE_VALID) +#define __WRITEABLE (_PAGE_DIRTY | _PAGE_WRITE) + +#define _PAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV) +#define _HPAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV | _PAGE_HUGE) + +#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_NO_READ | \ + _PAGE_USER | _CACHE_CC) +#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \ + _PAGE_USER | _CACHE_CC) +#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _CACHE_CC) + +#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_CC) +#define PAGE_KERNEL_SUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC) +#define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC) + +#define __P000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) +#define __P001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) +#define __P010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) +#define __P011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) +#define __P100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) +#define __P101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) +#define __P110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) +#define __P111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) + +#define __S000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) +#define __S001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) +#define __S010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) +#define __S011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) +#define __S100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) +#define __S101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) +#define __S110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) +#define __S111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) + +#ifndef __ASSEMBLY__ + +#define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL_SUC) + +#define pgprot_noncached pgprot_noncached + +static inline pgprot_t pgprot_noncached(pgprot_t _prot) +{ + unsigned long prot = pgprot_val(_prot); + + prot = (prot & ~_CACHE_MASK) | _CACHE_SUC; + + return __pgprot(prot); +} + +extern bool wc_enabled; + +#define pgprot_writecombine pgprot_writecombine + +static inline pgprot_t pgprot_writecombine(pgprot_t _prot) +{ + unsigned long prot = pgprot_val(_prot); + + prot = (prot & ~_CACHE_MASK) | (wc_enabled ? _CACHE_WUC : _CACHE_SUC); + + return __pgprot(prot); +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_PGTABLE_BITS_H */ diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h new file mode 100644 index 0000000000000000000000000000000000000000..24157f778baaa160057a70182d410636f95a3e35 --- /dev/null +++ b/arch/loongarch/include/asm/pgtable.h @@ -0,0 +1,336 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PGTABLE_H +#define _ASM_PGTABLE_H + +#include +#include +#include +#include + +struct mm_struct; +struct vm_area_struct; + +/* + * ZERO_PAGE is a global shared page that is always zero; used + * for zero-mapped memory areas etc.. + */ + +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; + +#define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page) + +extern void paging_init(void); + +#define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) +#define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE)) +#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC) + +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; + if (pte_val(pteval) & _PAGE_GLOBAL) { + pte_t *buddy = ptep_buddy(ptep); + /* + * Make sure the buddy is global too (if it's !none, + * it better already be global) + */ +#ifdef CONFIG_SMP + /* + * For SMP, multiple CPUs can race, so we need to do + * this atomically. + */ + unsigned long page_global = _PAGE_GLOBAL; + unsigned long tmp; + + __asm__ __volatile__ ( + "1:" __LL "%[tmp], %[buddy] \n" + " bnez %[tmp], 2f \n" + " or %[tmp], %[tmp], %[global] \n" + __SC "%[tmp], %[buddy] \n" + " beqz %[tmp], 1b \n" + " nop \n" + "2: \n" + __WEAK_LLSC_MB + : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp) + : [global] "r" (page_global)); +#else /* !CONFIG_SMP */ + if (pte_none(*buddy)) + pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL; +#endif /* CONFIG_SMP */ + } +} + +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + set_pte(ptep, pteval); +} + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + /* Preserve global status for the pair */ + if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL) + set_pte_at(mm, addr, ptep, __pte(_PAGE_GLOBAL)); + else + set_pte_at(mm, addr, ptep, __pte(0)); +} + +#define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1) +#define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1) +#define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1) + +extern pgd_t swapper_pg_dir[]; +extern pgd_t invalid_pg_dir[]; + +/* + * The following only work if pte_present() is true. + * Undefined behaviour if not.. + */ +static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } +static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } +static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); } + +static inline pte_t pte_mkold(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_ACCESSED; + return pte; +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + pte_val(pte) |= _PAGE_ACCESSED; + return pte; +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); + return pte; +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + pte_val(pte) |= _PAGE_MODIFIED; + if (pte_val(pte) & _PAGE_WRITE) + pte_val(pte) |= _PAGE_DIRTY; + return pte; +} + +static inline pte_t pte_mkwrite(pte_t pte) +{ + pte_val(pte) |= _PAGE_WRITE; + if (pte_val(pte) & _PAGE_MODIFIED) + pte_val(pte) |= _PAGE_DIRTY; + return pte; +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY); + return pte; +} + +static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; } + +static inline pte_t pte_mkhuge(pte_t pte) +{ + pte_val(pte) |= _PAGE_HUGE; + return pte; +} + +#if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) +static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } +static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; } +#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ + +#define pte_accessible pte_accessible +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) +{ + if (pte_val(a) & _PAGE_PRESENT) + return true; + + if ((pte_val(a) & _PAGE_PROTNONE) && + mm_tlb_flush_pending(mm)) + return true; + + return false; +} + +/* + * Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + */ +#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | + (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); +} + +extern void __update_tlb(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); + +static inline void update_mmu_cache(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + __update_tlb(vma, address, ptep); +} + +#define __HAVE_ARCH_UPDATE_MMU_TLB +#define update_mmu_tlb update_mmu_cache + +static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + __update_tlb(vma, address, (pte_t *)pmdp); +} + +#define kern_addr_valid(addr) (1) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/ +#define pmdp_establish generic_pmdp_establish + +static inline int pmd_trans_huge(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd); +} + +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) | + ((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); + pmd_val(pmd) |= _PAGE_HUGE; + + return pmd; +} + +#define pmd_write pmd_write +static inline int pmd_write(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_WRITE); +} + +static inline pmd_t pmd_mkwrite(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_WRITE; + if (pmd_val(pmd) & _PAGE_MODIFIED) + pmd_val(pmd) |= _PAGE_DIRTY; + return pmd; +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY); + return pmd; +} + +static inline int pmd_dirty(pmd_t pmd) +{ + return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED)); +} + +static inline pmd_t pmd_mkclean(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); + return pmd; +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_MODIFIED; + if (pmd_val(pmd) & _PAGE_WRITE) + pmd_val(pmd) |= _PAGE_DIRTY; + return pmd; +} + +static inline int pmd_young(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_ACCESSED); +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_ACCESSED; + return pmd; +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_ACCESSED; + return pmd; +} + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return (pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT; +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + if (pmd_trans_huge(pmd)) + return pfn_to_page(pmd_pfn(pmd)); + + return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | + (pgprot_val(newprot) & ~_HPAGE_CHG_MASK); + return pmd; +} + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_PRESENT_INVALID; + pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE); + + return pmd; +} + +/* + * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a + * different prototype. + */ +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + pmd_t old = *pmdp; + + pmd_clear(pmdp); + + return old; +} + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifdef CONFIG_NUMA_BALANCING +static inline long pte_protnone(pte_t pte) +{ + return (pte_val(pte) & _PAGE_PROTNONE); +} + +static inline long pmd_protnone(pmd_t pmd) +{ + return (pmd_val(pmd) & _PAGE_PROTNONE); +} +#endif /* CONFIG_NUMA_BALANCING */ + +#define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0) +#define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0) + +/* + * We provide our own get_unmapped area to cope with the virtual aliasing + * constraints placed on us by the cache architecture. + */ +#define HAVE_ARCH_UNMAPPED_AREA +#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN + +#endif /* _ASM_PGTABLE_H */ diff --git a/arch/loongarch/include/asm/prefetch.h b/arch/loongarch/include/asm/prefetch.h new file mode 100644 index 0000000000000000000000000000000000000000..47169b57ce32595b7e84b70d51631a7ce6b2e43e --- /dev/null +++ b/arch/loongarch/include/asm/prefetch.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef __ASM_PREFETCH_H +#define __ASM_PREFETCH_H + +#define Pref_Load 0 +#define Pref_Store 8 + +#ifdef __ASSEMBLY__ + + .macro __pref hint addr +#ifdef CONFIG_CPU_HAS_PREFETCH + preld \hint, \addr, 0 +#endif + .endm + + .macro pref_load addr + __pref Pref_Load, \addr + .endm + + .macro pref_store addr + __pref Pref_Store, \addr + .endm + +#endif + +#endif /* __ASM_PREFETCH_H */ diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h new file mode 100644 index 0000000000000000000000000000000000000000..0e86c8a1898966175501795777ebd04ba73701bb --- /dev/null +++ b/arch/loongarch/include/asm/processor.h @@ -0,0 +1,238 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_PROCESSOR_H +#define _ASM_PROCESSOR_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_32BIT + +#define TASK_SIZE 0x80000000UL +#define TASK_SIZE_MIN TASK_SIZE +#define STACK_TOP_MAX TASK_SIZE + +#define TASK_IS_32BIT_ADDR 1 + +#endif + +#ifdef CONFIG_64BIT + +#define TASK_SIZE32 0x100000000UL +#define TASK_SIZE64 (0x1UL << ((cpu_vabits > VA_BITS) ? VA_BITS : cpu_vabits)) + +#define TASK_SIZE (test_thread_flag(TIF_32BIT_ADDR) ? TASK_SIZE32 : TASK_SIZE64) +#define TASK_SIZE_MIN TASK_SIZE32 +#define STACK_TOP_MAX TASK_SIZE64 + +#define TASK_SIZE_OF(tsk) \ + (test_tsk_thread_flag(tsk, TIF_32BIT_ADDR) ? TASK_SIZE32 : TASK_SIZE64) + +#define TASK_IS_32BIT_ADDR test_thread_flag(TIF_32BIT_ADDR) + +#endif + +#define VDSO_RANDOMIZE_SIZE (TASK_IS_32BIT_ADDR ? SZ_1M : SZ_64M) + +unsigned long stack_top(void); +#define STACK_TOP stack_top() + +/* + * This decides where the kernel will search for a free chunk of vm + * space during mmap's. + */ +#define TASK_UNMAPPED_BASE PAGE_ALIGN(TASK_SIZE / 3) + +#define FPU_REG_WIDTH 256 +#define FPU_ALIGN __attribute__((aligned(32))) + +union fpureg { + __u32 val32[FPU_REG_WIDTH / 32]; + __u64 val64[FPU_REG_WIDTH / 64]; +}; + +#define FPR_IDX(width, idx) (idx) + +#define BUILD_FPR_ACCESS(width) \ +static inline u##width get_fpr##width(union fpureg *fpr, unsigned idx) \ +{ \ + return fpr->val##width[FPR_IDX(width, idx)]; \ +} \ + \ +static inline void set_fpr##width(union fpureg *fpr, unsigned idx, \ + u##width val) \ +{ \ + fpr->val##width[FPR_IDX(width, idx)] = val; \ +} + +BUILD_FPR_ACCESS(32) +BUILD_FPR_ACCESS(64) + +struct loongarch_fpu { + unsigned int fcsr; + uint64_t fcc; /* 8x8 */ + uint64_t ftop; + union fpureg fpr[NUM_FPU_REGS]; +}; + +struct loongarch_lbt { + /* Scratch registers */ + unsigned long scr0; + unsigned long scr1; + unsigned long scr2; + unsigned long scr3; + /* Eflags register */ + unsigned long eflags; +}; + +#define INIT_CPUMASK { \ + {0,} \ +} + +struct loongarch_watch_reg_state { + unsigned long addr[NUM_WATCH_REGS]; + unsigned long mask[NUM_WATCH_REGS]; + unsigned char irw[NUM_WATCH_REGS]; + unsigned char irwstat[NUM_WATCH_REGS]; + unsigned char irwmask[NUM_WATCH_REGS]; +}; + +#define ARCH_MIN_TASKALIGN 32 + +struct loongarch_vdso_info; + +/* + * If you change thread_struct remember to change the #defines below too! + */ +struct thread_struct { + /* Main processor registers. */ + unsigned long reg01, reg03, reg22; /* ra sp fp */ + unsigned long reg23, reg24, reg25, reg26; /* s0-s3 */ + unsigned long reg27, reg28, reg29, reg30, reg31; /* s4-s8 */ + + /* __schedule() return address / call frame address */ + unsigned long sched_ra; + unsigned long sched_cfa; + + /* CSR registers */ + unsigned long csr_prmd; + unsigned long csr_crmd; + unsigned long csr_euen; + unsigned long csr_ecfg; + unsigned long csr_badvaddr; /* Last user fault */ + + /* Used by ptrace single_step */ + unsigned long single_step; + + /* Watch register state, if available. */ + struct loongarch_watch_reg_state watch; + + /* Other stuff associated with the thread. */ + unsigned long trap_nr; + unsigned long error_code; + struct loongarch_lbt lbt; + struct loongarch_vdso_info *vdso; + + /* + * FPU & vector registers, must be at last because + * they are conditionally copied at fork(). + */ + struct loongarch_fpu fpu FPU_ALIGN; +}; + +#define thread_saved_ra(tsk) (tsk->thread.sched_ra) +#define thread_saved_fp(tsk) (tsk->thread.sched_cfa) + +#define INIT_THREAD { \ + /* \ + * Main processor registers \ + */ \ + .reg01 = 0, \ + .reg03 = 0, \ + .reg22 = 0, \ + .reg23 = 0, \ + .reg24 = 0, \ + .reg25 = 0, \ + .reg26 = 0, \ + .reg27 = 0, \ + .reg28 = 0, \ + .reg29 = 0, \ + .reg30 = 0, \ + .reg31 = 0, \ + .sched_ra = 0, \ + .sched_cfa = 0, \ + .csr_crmd = 0, \ + .csr_prmd = 0, \ + .csr_euen = 0, \ + .csr_ecfg = 0, \ + .csr_badvaddr = 0, \ + /* \ + * Saved watch register stuff \ + */ \ + .watch = {{0,},}, \ + /* \ + * Other stuff associated with the process \ + */ \ + .trap_nr = 0, \ + .error_code = 0, \ + /* \ + * FPU & vector registers \ + */ \ + .fpu = { \ + .fcsr = 0, \ + .fcc = 0, \ + .fpr = {{{0,},},}, \ + }, \ +} + +struct task_struct; + +/* Free all resources held by a thread. */ +#define release_thread(thread) do { } while(0) + +enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, + IDLE_POLL}; + +extern unsigned long boot_option_idle_override; +/* + * Do necessary setup to start up a newly executed thread. + */ +extern void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp); + +static inline void flush_thread(void) +{ +} + +unsigned long get_wchan(struct task_struct *p); + +#define __KSTK_TOS(tsk) ((unsigned long)task_stack_page(tsk) + \ + THREAD_SIZE - sizeof(struct pt_regs)) +#define task_pt_regs(tsk) ((struct pt_regs *)__KSTK_TOS(tsk)) +#define KSTK_EIP(tsk) (task_pt_regs(tsk)->csr_era) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[3]) +#define KSTK_EUEN(tsk) (task_pt_regs(tsk)->csr_euen) +#define KSTK_ECFG(tsk) (task_pt_regs(tsk)->csr_ecfg) + +#define return_address() ({__asm__ __volatile__("":::"$1");__builtin_return_address(0);}) + +#ifdef CONFIG_CPU_HAS_PREFETCH + +#define ARCH_HAS_PREFETCH +#define prefetch(x) __builtin_prefetch((x), 0, 1) + +#define ARCH_HAS_PREFETCHW +#define prefetchw(x) __builtin_prefetch((x), 1, 1) + +#endif + +#endif /* _ASM_PROCESSOR_H */ diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h new file mode 100644 index 0000000000000000000000000000000000000000..095479abef46b0834e2632079a87e08b78042e75 --- /dev/null +++ b/arch/loongarch/include/asm/ptrace.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_PTRACE_H +#define _ASM_PTRACE_H + +#include +#include +#include +#include + +/* + * This struct defines the way the registers are stored on the stack during + * a system call/exception. If you add a register here, please also add it to + * regoffset_table[] in arch/loongarch/kernel/ptrace.c. + */ +struct pt_regs { + /* Main processor registers. */ + unsigned long regs[32]; + + /* Original syscall arg0. */ + unsigned long orig_a0; + + /* Special CSR registers. */ + unsigned long csr_era; + unsigned long csr_badvaddr; + unsigned long csr_crmd; + unsigned long csr_prmd; + unsigned long csr_euen; + unsigned long csr_ecfg; + unsigned long csr_estat; + unsigned long __last[]; +} __aligned(8); + +static inline int regs_irqs_disabled(struct pt_regs *regs) +{ + return arch_irqs_disabled_flags(regs->csr_prmd); +} + +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->regs[3]; +} + +/* + * Don't use asm-generic/ptrace.h it defines FP accessors that don't make + * sense on LoongArch. We rather want an error if they get invoked. + */ + +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->csr_era = val; +} + +/* Query offset/name of register from its name/offset */ +extern int regs_query_register_offset(const char *name); +#define MAX_REG_OFFSET (offsetof(struct pt_regs, __last)) + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten. + * @offset: offset number of the register. + * + * regs_get_register returns the value of a register. The @offset is the + * offset of the register in struct pt_regs address which specified by @regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static inline int regs_within_kernel_stack(struct pt_regs *regs, + unsigned long addr) +{ + return ((addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + +struct task_struct; + +/* + * Does the process account for user or for system time? + */ +#define user_mode(regs) (((regs)->csr_prmd & PLV_MASK) == PLV_USER) + +static inline long regs_return_value(struct pt_regs *regs) +{ + return regs->regs[4]; +} + +#define instruction_pointer(regs) ((regs)->csr_era) +#define profile_pc(regs) instruction_pointer(regs) + +extern void die(const char *, struct pt_regs *) __noreturn; + +static inline void die_if_kernel(const char *str, struct pt_regs *regs) +{ + if (unlikely(!user_mode(regs))) + die(str, regs); +} + +#define current_pt_regs() \ +({ \ + unsigned long sp = (unsigned long)__builtin_frame_address(0); \ + (struct pt_regs *)((sp | (THREAD_SIZE - 1)) + 1) - 1; \ +}) + +/* Helpers for working with the user stack pointer */ + +static inline unsigned long user_stack_pointer(struct pt_regs *regs) +{ + return regs->regs[3]; +} + +static inline void user_stack_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->regs[3] = val; +} + +#define arch_has_single_step() (1) + +#endif /* _ASM_PTRACE_H */ diff --git a/arch/loongarch/include/asm/qspinlock.h b/arch/loongarch/include/asm/qspinlock.h new file mode 100644 index 0000000000000000000000000000000000000000..3ef3e784774113fea98201534edfe3bf6338fca7 --- /dev/null +++ b/arch/loongarch/include/asm/qspinlock.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_QSPINLOCK_H +#define _ASM_QSPINLOCK_H + +#include +#include + +#define _Q_PENDING_LOOPS (1 << 9) +#define queued_spin_unlock queued_spin_unlock + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __pv_init_lock_hash(void); +extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __pv_queued_spin_unlock(struct qspinlock *lock); + +static inline void native_queued_spin_unlock(struct qspinlock *lock) +{ + compiletime_assert_atomic_type(lock->locked); + c_sync(); + WRITE_ONCE(lock->locked, 0); +} + +static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + pv_queued_spin_lock_slowpath(lock, val); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + pv_queued_spin_unlock(lock); +} + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted(long cpu) +{ + return pv_vcpu_is_preempted(cpu); +} +#else +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + compiletime_assert_atomic_type(lock->locked); + c_sync(); + WRITE_ONCE(lock->locked, 0); +} +#endif + +#include + +#endif /* _ASM_QSPINLOCK_H */ diff --git a/arch/loongarch/include/asm/qspinlock_paravirt.h b/arch/loongarch/include/asm/qspinlock_paravirt.h new file mode 100644 index 0000000000000000000000000000000000000000..7d03855961274d0835803230b5f6e0e6e44cd620 --- /dev/null +++ b/arch/loongarch/include/asm/qspinlock_paravirt.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_QSPINLOCK_PARAVIRT_H +#define __ASM_QSPINLOCK_PARAVIRT_H + +#ifdef CONFIG_64BIT + +#define __pv_queued_spin_unlock __pv_queued_spin_unlock +void __pv_queued_spin_unlock(struct qspinlock *lock) +{ + u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); + + if (likely(lockval == _Q_LOCKED_VAL)) + return; + __pv_queued_spin_unlock_slowpath(lock, lockval); +} + +#endif /* CONFIG_64BIT */ +#endif diff --git a/arch/loongarch/include/asm/reboot.h b/arch/loongarch/include/asm/reboot.h new file mode 100644 index 0000000000000000000000000000000000000000..48c3b1ea7145fb46f0b896fd190cfaa5c220abdd --- /dev/null +++ b/arch/loongarch/include/asm/reboot.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_REBOOT_H +#define _ASM_REBOOT_H + +extern void (*pm_restart)(void); + +#endif /* _ASM_REBOOT_H */ diff --git a/arch/loongarch/include/asm/regdef.h b/arch/loongarch/include/asm/regdef.h new file mode 100644 index 0000000000000000000000000000000000000000..d6f09dac861c89632e07392ebad8d6c3ecfc01ba --- /dev/null +++ b/arch/loongarch/include/asm/regdef.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_REGDEF_H +#define _ASM_REGDEF_H + +#define zero $r0 /* wired zero */ +#define ra $r1 /* return address */ +#define tp $r2 +#define sp $r3 /* stack pointer */ +#define a0 $r4 /* argument registers, a0/a1 reused as v0/v1 for return value */ +#define a1 $r5 +#define a2 $r6 +#define a3 $r7 +#define a4 $r8 +#define a5 $r9 +#define a6 $r10 +#define a7 $r11 +#define t0 $r12 /* caller saved */ +#define t1 $r13 +#define t2 $r14 +#define t3 $r15 +#define t4 $r16 +#define t5 $r17 +#define t6 $r18 +#define t7 $r19 +#define t8 $r20 +#define u0 $r21 +#define fp $r22 /* frame pointer */ +#define s0 $r23 /* callee saved */ +#define s1 $r24 +#define s2 $r25 +#define s3 $r26 +#define s4 $r27 +#define s5 $r28 +#define s6 $r29 +#define s7 $r30 +#define s8 $r31 + +#endif /* _ASM_REGDEF_H */ diff --git a/arch/loongarch/include/asm/serial.h b/arch/loongarch/include/asm/serial.h new file mode 100644 index 0000000000000000000000000000000000000000..f0bead2e9995e8fde31c42cf36603991035a0290 --- /dev/null +++ b/arch/loongarch/include/asm/serial.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef __ASM__SERIAL_H +#define __ASM__SERIAL_H + +#define BASE_BAUD 0 +#define STD_COM_FLAGS (ASYNC_BOOT_AUTOCONF | ASYNC_SKIP_TEST) + +#endif /* __ASM__SERIAL_H */ diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h new file mode 100644 index 0000000000000000000000000000000000000000..7628c51616d6ea52494b433f8f8a5c27edcef837 --- /dev/null +++ b/arch/loongarch/include/asm/setup.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#ifndef _LOONGARCH_SETUP_H +#define _LOONGARCH_SETUP_H + +#include +#include + +#define VECSIZE 0x200 + +extern unsigned long eentry; +extern unsigned long tlbrentry; +extern void tlb_init(int cpu); +extern void cpu_cache_init(void); +extern void cache_error_setup(void); +extern void per_cpu_trap_init(int cpu); +extern void set_handler(unsigned long offset, void *addr, unsigned long len); +extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len); + +#ifdef CONFIG_USE_OF + +struct boot_param_header; + +extern void device_tree_init(void); + +#else /* CONFIG_OF */ +static inline void device_tree_init(void) { } +#endif /* CONFIG_OF */ + +#endif /* __SETUP_H */ diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h new file mode 100644 index 0000000000000000000000000000000000000000..340e88d09f2c356d4e975e002f4162c800585ba8 --- /dev/null +++ b/arch/loongarch/include/asm/smp.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_SMP_H +#define __ASM_SMP_H + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_SMP + +struct task_struct; + +struct plat_smp_ops { + void (*send_ipi_single)(int cpu, unsigned int action); + void (*send_ipi_mask)(const struct cpumask *mask, unsigned int action); + void (*smp_setup)(void); + void (*prepare_cpus)(unsigned int max_cpus); + int (*boot_secondary)(int cpu, struct task_struct *idle); + void (*init_secondary)(void); + void (*smp_finish)(void); +#ifdef CONFIG_HOTPLUG_CPU + int (*cpu_disable)(void); + void (*cpu_die)(unsigned int cpu); +#endif +}; + +extern struct plat_smp_ops *mp_ops; +void register_smp_ops(const struct plat_smp_ops *ops); + +static inline void plat_smp_setup(void) +{ + mp_ops->smp_setup(); +} + +#else /* !CONFIG_SMP */ + +struct plat_smp_ops; + +static inline void plat_smp_setup(void) +{ + /* UP, nothing to do ... */ +} + +static inline void register_smp_ops(const struct plat_smp_ops *ops) +{ +} + +#endif /* !CONFIG_SMP */ + +extern int smp_num_siblings; +extern int num_processors; +extern int disabled_cpus; +extern cpumask_t cpu_sibling_map[]; +extern cpumask_t cpu_core_map[]; +extern cpumask_t cpu_foreign_map[]; + +static inline int raw_smp_processor_id(void) +{ +#if defined(__VDSO__) + extern int vdso_smp_processor_id(void) + __compiletime_error("VDSO should not call smp_processor_id()"); + return vdso_smp_processor_id(); +#else + return current_thread_info()->cpu; +#endif +} +#define raw_smp_processor_id raw_smp_processor_id + +/* Map from cpu id to sequential logical cpu number. This will only + not be idempotent when cpus failed to come on-line. */ +extern int __cpu_number_map[NR_CPUS]; +#define cpu_number_map(cpu) __cpu_number_map[cpu] + +/* The reverse map from sequential logical cpu number to cpu id. */ +extern int __cpu_logical_map[NR_CPUS]; +#define cpu_logical_map(cpu) __cpu_logical_map[cpu] + +#define cpu_physical_id(cpu) cpu_logical_map(cpu) + +#define SMP_BOOT_CPU 0x1 +#define SMP_RESCHEDULE 0x2 +#define SMP_CALL_FUNCTION 0x4 + +struct secondary_data { + unsigned long stack; + unsigned long thread_info; +}; +extern struct secondary_data cpuboot_data; + +extern asmlinkage void smpboot_entry(void); +extern asmlinkage void start_secondary(void); + +extern void set_cpu_sibling_map(int cpu); +extern void clear_cpu_sibling_map(int cpu); +extern void calculate_cpu_foreign_map(void); + +/* + * Generate IPI list text + */ +extern void show_ipi_list(struct seq_file *p, int prec); + +/* + * this function sends a 'reschedule' IPI to another CPU. + * it goes straight through and wastes no time serializing + * anything. Worst case is that we lose a reschedule ... + */ +static inline void smp_send_reschedule(int cpu) +{ + mp_ops->send_ipi_single(cpu, SMP_RESCHEDULE); +} + +#ifdef CONFIG_HOTPLUG_CPU +static inline int __cpu_disable(void) +{ + return mp_ops->cpu_disable(); +} + +static inline void __cpu_die(unsigned int cpu) +{ + mp_ops->cpu_die(cpu); +} +#endif + +static inline void arch_send_call_function_single_ipi(int cpu) +{ + mp_ops->send_ipi_single(cpu, SMP_CALL_FUNCTION); +} + +static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION); +} + +#endif /* __ASM_SMP_H */ diff --git a/arch/loongarch/include/asm/sparsemem.h b/arch/loongarch/include/asm/sparsemem.h new file mode 100644 index 0000000000000000000000000000000000000000..fd39eea7a37510f97ee704def987e6fcc2099eed --- /dev/null +++ b/arch/loongarch/include/asm/sparsemem.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LOONGARCH_SPARSEMEM_H +#define _LOONGARCH_SPARSEMEM_H + +#ifdef CONFIG_SPARSEMEM + +/* + * SECTION_SIZE_BITS 2^N: how big each section will be + * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space + */ +#define SECTION_SIZE_BITS 29 /* 2^29 = Largest Huge Page Size */ +#define MAX_PHYSMEM_BITS 48 + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +#define VMEMMAP_SIZE (sizeof(struct page) * (1UL << (cpu_pabits + 1 - PAGE_SHIFT))) +#endif + +#include + +#endif /* CONFIG_SPARSEMEM */ + +#ifndef VMEMMAP_SIZE +#define VMEMMAP_SIZE 0 /* 1, For FLATMEM; 2, For SPARSEMEM without VMEMMAP. */ +#endif + +#ifdef CONFIG_MEMORY_HOTPLUG +int memory_add_physaddr_to_nid(u64 addr); +#define memory_add_physaddr_to_nid memory_add_physaddr_to_nid +#endif + +#define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS) + +#endif /* _LOONGARCH_SPARSEMEM_H */ diff --git a/arch/loongarch/include/asm/spinlock.h b/arch/loongarch/include/asm/spinlock.h new file mode 100644 index 0000000000000000000000000000000000000000..60df512425238bca5d97f338f2b55de070ac5ae1 --- /dev/null +++ b/arch/loongarch/include/asm/spinlock.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SPINLOCK_H +#define _ASM_SPINLOCK_H + +/* How long a lock should spin before we consider blocking */ +#define SPIN_THRESHOLD (1 << 15) + +#include +#include +#include + +#endif /* _ASM_SPINLOCK_H */ diff --git a/arch/loongarch/include/asm/spinlock_types.h b/arch/loongarch/include/asm/spinlock_types.h new file mode 100644 index 0000000000000000000000000000000000000000..1cbfe2451529b48d4ae33174ee7c2eb00c4f7be5 --- /dev/null +++ b/arch/loongarch/include/asm/spinlock_types.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SPINLOCK_TYPES_H +#define _ASM_SPINLOCK_TYPES_H + +#include +#include + +#endif diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h new file mode 100644 index 0000000000000000000000000000000000000000..c4895cc9003dad3b30fddd921d40e73dd8ee5017 --- /dev/null +++ b/arch/loongarch/include/asm/stackframe.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_STACKFRAME_H +#define _ASM_STACKFRAME_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +/* Make the addition of cfi info a little easier. */ + .macro cfi_rel_offset reg offset=0 docfi=0 + .if \docfi + .cfi_rel_offset \reg, \offset + .endif + .endm + + .macro cfi_st reg offset=0 docfi=0 + cfi_rel_offset \reg, \offset, \docfi + LONG_S \reg, sp, \offset + .endm + + .macro cfi_restore reg offset=0 docfi=0 + .if \docfi + .cfi_restore \reg + .endif + .endm + + .macro cfi_ld reg offset=0 docfi=0 + LONG_L \reg, sp, \offset + cfi_restore \reg \offset \docfi + .endm + +/* Jump to the runtime virtual address. */ + .macro JUMP_VIRT_ADDR temp1 temp2 + li.d \temp1, CACHE_BASE + pcaddi \temp2, 0 + or \temp1, \temp1, \temp2 + jirl zero, \temp1, 0xc + .endm + + .macro BACKUP_T0T1 + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + .endm + + .macro RELOAD_T0T1 + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + .endm + + .macro SAVE_TEMP docfi=0 + RELOAD_T0T1 + cfi_st t0, PT_R12, \docfi + cfi_st t1, PT_R13, \docfi + cfi_st t2, PT_R14, \docfi + cfi_st t3, PT_R15, \docfi + cfi_st t4, PT_R16, \docfi + cfi_st t5, PT_R17, \docfi + cfi_st t6, PT_R18, \docfi + cfi_st t7, PT_R19, \docfi + cfi_st t8, PT_R20, \docfi + .endm + + .macro SAVE_STATIC docfi=0 + cfi_st s0, PT_R23, \docfi + cfi_st s1, PT_R24, \docfi + cfi_st s2, PT_R25, \docfi + cfi_st s3, PT_R26, \docfi + cfi_st s4, PT_R27, \docfi + cfi_st s5, PT_R28, \docfi + cfi_st s6, PT_R29, \docfi + cfi_st s7, PT_R30, \docfi + cfi_st s8, PT_R31, \docfi + .endm + +/* + * get_saved_sp returns the SP for the current CPU by looking in the + * kernelsp array for it. It stores the current sp in t0 and loads the + * new value in sp. + */ + .macro get_saved_sp docfi=0 + la.abs t1, kernelsp +#ifdef CONFIG_SMP + csrrd t0, PERCPU_BASE_KS + LONG_ADD t1, t1, t0 +#endif + move t0, sp + .if \docfi + .cfi_register sp, t0 + .endif + LONG_L sp, t1, 0 + .endm + + .macro set_saved_sp stackp temp temp2 + la.abs \temp, kernelsp +#ifdef CONFIG_SMP + LONG_ADD \temp, \temp, u0 +#endif + LONG_S \stackp, \temp, 0 + .endm + + .macro SAVE_SOME docfi=0 + csrrd t1, LOONGARCH_CSR_PRMD + andi t1, t1, 0x3 /* extract pplv bit */ + move t0, sp + beqz t1, 8f + /* Called from user mode, new stack. */ + get_saved_sp docfi=\docfi +8: + PTR_ADDI sp, sp, -PT_SIZE + .if \docfi + .cfi_def_cfa sp,0 + .endif + cfi_st t0, PT_R3, \docfi + cfi_rel_offset sp, PT_R3, \docfi + LONG_S zero, sp, PT_R0 + csrrd t0, LOONGARCH_CSR_PRMD + LONG_S t0, sp, PT_PRMD + csrrd t0, LOONGARCH_CSR_CRMD + LONG_S t0, sp, PT_CRMD + csrrd t0, LOONGARCH_CSR_EUEN + LONG_S t0, sp, PT_EUEN + csrrd t0, LOONGARCH_CSR_ECFG + LONG_S t0, sp, PT_ECFG + csrrd t0, LOONGARCH_CSR_ESTAT + PTR_S t0, sp, PT_ESTAT + cfi_st ra, PT_R1, \docfi + cfi_st a0, PT_R4, \docfi + cfi_st a1, PT_R5, \docfi + cfi_st a2, PT_R6, \docfi + cfi_st a3, PT_R7, \docfi + cfi_st a4, PT_R8, \docfi + cfi_st a5, PT_R9, \docfi + cfi_st a6, PT_R10, \docfi + cfi_st a7, PT_R11, \docfi + csrrd ra, LOONGARCH_CSR_ERA + LONG_S ra, sp, PT_ERA + .if \docfi + .cfi_rel_offset ra, PT_ERA + .endif + cfi_st tp, PT_R2, \docfi + cfi_st fp, PT_R22, \docfi + + /* Set thread_info if we're coming from user mode */ + csrrd t0, LOONGARCH_CSR_PRMD + andi t0, t0, 0x3 /* extract pplv bit */ + beqz t0, 9f + + li.d tp, ~_THREAD_MASK + and tp, tp, sp + cfi_st u0, PT_R21, \docfi + csrrd u0, PERCPU_BASE_KS +9: +#ifdef CONFIG_KGDB + li.w t0, CSR_CRMD_WE + csrxchg t0, t0, LOONGARCH_CSR_CRMD +#endif + UNWIND_HINT_REGS + .endm + + .macro SAVE_ALL docfi=0 + SAVE_SOME \docfi + SAVE_TEMP \docfi + SAVE_STATIC \docfi + .endm + + .macro RESTORE_TEMP docfi=0 + cfi_ld t0, PT_R12, \docfi + cfi_ld t1, PT_R13, \docfi + cfi_ld t2, PT_R14, \docfi + cfi_ld t3, PT_R15, \docfi + cfi_ld t4, PT_R16, \docfi + cfi_ld t5, PT_R17, \docfi + cfi_ld t6, PT_R18, \docfi + cfi_ld t7, PT_R19, \docfi + cfi_ld t8, PT_R20, \docfi + .endm + + .macro RESTORE_STATIC docfi=0 + cfi_ld s0, PT_R23, \docfi + cfi_ld s1, PT_R24, \docfi + cfi_ld s2, PT_R25, \docfi + cfi_ld s3, PT_R26, \docfi + cfi_ld s4, PT_R27, \docfi + cfi_ld s5, PT_R28, \docfi + cfi_ld s6, PT_R29, \docfi + cfi_ld s7, PT_R30, \docfi + cfi_ld s8, PT_R31, \docfi + .endm + + .macro RESTORE_SOME docfi=0 + LONG_L a0, sp, PT_PRMD + andi a0, a0, 0x3 /* extract pplv bit */ + beqz a0, 8f + cfi_ld u0, PT_R21, \docfi +8: + LONG_L a0, sp, PT_ERA + csrwr a0, LOONGARCH_CSR_ERA + LONG_L a0, sp, PT_PRMD + csrwr a0, LOONGARCH_CSR_PRMD + cfi_ld ra, PT_R1, \docfi + cfi_ld a0, PT_R4, \docfi + cfi_ld a1, PT_R5, \docfi + cfi_ld a2, PT_R6, \docfi + cfi_ld a3, PT_R7, \docfi + cfi_ld a4, PT_R8, \docfi + cfi_ld a5, PT_R9, \docfi + cfi_ld a6, PT_R10, \docfi + cfi_ld a7, PT_R11, \docfi + cfi_ld tp, PT_R2, \docfi + cfi_ld fp, PT_R22, \docfi + .endm + + .macro RESTORE_SP_AND_RET docfi=0 + cfi_ld sp, PT_R3, \docfi + UNWIND_HINT sp_reg=ORC_REG_SP type=ORC_TYPE_CALL + ertn + .endm + + .macro RESTORE_ALL_AND_RET docfi=0 + RESTORE_STATIC \docfi + RESTORE_TEMP \docfi + RESTORE_SOME \docfi + RESTORE_SP_AND_RET \docfi + .endm + +#endif /* _ASM_STACKFRAME_H */ diff --git a/arch/loongarch/include/asm/stackprotector.h b/arch/loongarch/include/asm/stackprotector.h new file mode 100644 index 0000000000000000000000000000000000000000..c51813a181fb0bc852532032a63337b20aba5012 --- /dev/null +++ b/arch/loongarch/include/asm/stackprotector.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GCC stack protector support. + * + * Stack protector works by putting predefined pattern at the start of + * the stack frame and verifying that it hasn't been overwritten when + * returning from the function. The pattern is called stack canary + * and gcc expects it to be defined by a global variable called + * "__stack_chk_guard" on LoongArch. This unfortunately means that on SMP + * we cannot have a different canary value per task. + */ + +#ifndef _ASM_STACKPROTECTOR_H +#define _ASM_STACKPROTECTOR_H 1 + +#include +#include + +extern unsigned long __stack_chk_guard; + +/* + * Initialize the stackprotector canary value. + * + * NOTE: this must only be called from functions that never return, + * and it must always be inlined. + */ +static __always_inline void boot_init_stack_canary(void) +{ + unsigned long canary; + + /* Try to get a semi random initial value. */ + get_random_bytes(&canary, sizeof(canary)); + canary ^= LINUX_VERSION_CODE; + + current->stack_canary = canary; + __stack_chk_guard = current->stack_canary; +} + +#endif /* _ASM_STACKPROTECTOR_H */ diff --git a/arch/loongarch/include/asm/stacktrace.h b/arch/loongarch/include/asm/stacktrace.h new file mode 100644 index 0000000000000000000000000000000000000000..4f27562d0e4b0e5c2fdfea1b38554887217987c0 --- /dev/null +++ b/arch/loongarch/include/asm/stacktrace.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ +#ifndef _ASM_STACKTRACE_H +#define _ASM_STACKTRACE_H + +#include +#include +#include +#include + +enum stack_type { + STACK_TYPE_UNKNOWN, + STACK_TYPE_TASK, + STACK_TYPE_IRQ, +}; + +struct stack_info { + enum stack_type type; + unsigned long begin, end, next_sp; +}; + +struct stack_frame { + unsigned long fp; + unsigned long ra; +}; + +bool in_task_stack(unsigned long stack, struct task_struct *task, + struct stack_info *info); +bool in_irq_stack(unsigned long stack, struct stack_info *info); +int get_stack_info(unsigned long stack, struct task_struct *task, + struct stack_info *info); + +#define STR_LONG_L __stringify(LONG_L) +#define STR_LONG_S __stringify(LONG_S) +#define STR_LONGSIZE __stringify(LONGSIZE) + +#define STORE_ONE_REG(r) \ + STR_LONG_S " $r" __stringify(r)", %1, "STR_LONGSIZE"*"__stringify(r)"\n\t" + +#define CSRRD_ONE_REG(reg) \ + __stringify(csrrd) " %0, "__stringify(reg)"\n\t" + +static __always_inline void prepare_frametrace(struct pt_regs *regs) +{ + __asm__ __volatile__( + /* Save $r1 */ + STORE_ONE_REG(1) + /* Use $r1 to save PC */ + "pcaddi $r1, 0\n\t" + STR_LONG_S " $r1, %0\n\t" + /* Restore $r1 */ + STR_LONG_L " $r1, %1, "STR_LONGSIZE"\n\t" + STORE_ONE_REG(2) + STORE_ONE_REG(3) + STORE_ONE_REG(4) + STORE_ONE_REG(5) + STORE_ONE_REG(6) + STORE_ONE_REG(7) + STORE_ONE_REG(8) + STORE_ONE_REG(9) + STORE_ONE_REG(10) + STORE_ONE_REG(11) + STORE_ONE_REG(12) + STORE_ONE_REG(13) + STORE_ONE_REG(14) + STORE_ONE_REG(15) + STORE_ONE_REG(16) + STORE_ONE_REG(17) + STORE_ONE_REG(18) + STORE_ONE_REG(19) + STORE_ONE_REG(20) + STORE_ONE_REG(21) + STORE_ONE_REG(22) + STORE_ONE_REG(23) + STORE_ONE_REG(24) + STORE_ONE_REG(25) + STORE_ONE_REG(26) + STORE_ONE_REG(27) + STORE_ONE_REG(28) + STORE_ONE_REG(29) + STORE_ONE_REG(30) + STORE_ONE_REG(31) + : "=m" (regs->csr_era) + : "r" (regs->regs) + : "memory"); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_BADV) : "=r" (regs->csr_badvaddr)); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_CRMD) : "=r" (regs->csr_crmd)); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_PRMD) : "=r" (regs->csr_prmd)); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_EUEN) : "=r" (regs->csr_euen)); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_ECFG) : "=r" (regs->csr_ecfg)); + __asm__ __volatile__(CSRRD_ONE_REG(LOONGARCH_CSR_ESTAT) : "=r" (regs->csr_estat)); +} + +#endif /* _ASM_STACKTRACE_H */ diff --git a/arch/loongarch/include/asm/string.h b/arch/loongarch/include/asm/string.h new file mode 100644 index 0000000000000000000000000000000000000000..fa4c8e896f02fb2858ff41381e6860b3c8864431 --- /dev/null +++ b/arch/loongarch/include/asm/string.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_STRING_H +#define _ASM_STRING_H + +#define __HAVE_ARCH_MEMSET +extern void *memset(void *__s, int __c, size_t __count); +extern void *__memset(void *__s, int __c, size_t __count); + +#define __HAVE_ARCH_MEMCPY +extern void *memcpy(void *__to, __const__ void *__from, size_t __n); +extern void *__memcpy(void *__to, __const__ void *__from, size_t __n); + +#define __HAVE_ARCH_MEMMOVE +extern void *memmove(void *__dest, __const__ void *__src, size_t __n); +extern void *__memmove(void *__dest, __const__ void *__src, size_t __n); + +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +/* + * For files that are not instrumented (e.g. mm/slub.c) we + * should use not instrumented version of mem* functions. + */ + +#define memset(s, c, n) __memset(s, c, n) +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memmove(dst, src, len) __memmove(dst, src, len) + +#ifndef __NO_FORTIFY +#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */ +#endif + +#endif + +#endif /* _ASM_STRING_H */ diff --git a/arch/loongarch/include/asm/switch_to.h b/arch/loongarch/include/asm/switch_to.h new file mode 100644 index 0000000000000000000000000000000000000000..d22331460981d8e566f1f645d554d7a9b44be8a3 --- /dev/null +++ b/arch/loongarch/include/asm/switch_to.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SWITCH_TO_H +#define _ASM_SWITCH_TO_H + +#include +#include +#include +#include + +struct task_struct; + +/** + * __switch_to - switch execution of a task + * @prev: The task previously executed. + * @next: The task to begin executing. + * @next_ti: task_thread_info(next). + * @sched_ra: __schedule return address. + * @sched_cfa: __schedule call frame address. + * + * This function is used whilst scheduling to save the context of prev & load + * the context of next. Returns prev. + */ +extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev, + struct task_struct *next, struct thread_info *next_ti, + void *sched_ra, void *sched_cfa); + +/* + * For newly created kernel threads switch_to() will return to + * ret_from_kernel_thread, newly created user threads to ret_from_fork. + * That is, everything following __switch_to() will be skipped for new threads. + * So everything that matters to new threads should be placed before __switch_to(). + */ +#define switch_to(prev, next, last) \ +do { \ + lose_fpu_inatomic(1, prev); \ + lose_lbt_inatomic(1, prev); \ + __process_watch(prev, next); \ + (last) = __switch_to(prev, next, task_thread_info(next), \ + __builtin_return_address(0), __builtin_frame_address(0)); \ +} while (0) + +#endif /* _ASM_SWITCH_TO_H */ diff --git a/arch/loongarch/include/asm/syscall.h b/arch/loongarch/include/asm/syscall.h new file mode 100644 index 0000000000000000000000000000000000000000..f6af611525b913e81c87acea8162c9796bb3662c --- /dev/null +++ b/arch/loongarch/include/asm/syscall.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ + +#ifndef __ASM_LOONGARCH_SYSCALL_H +#define __ASM_LOONGARCH_SYSCALL_H + +#include +#include +#include +#include +#include +#include +#include +#include + +extern void *sys_call_table[]; + +static inline long syscall_get_nr(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->regs[11]; +} + +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + regs->regs[4] = regs->orig_a0; +} + +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + unsigned long error = regs->regs[4]; + + return IS_ERR_VALUE(error) ? error : 0; +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->regs[4]; +} + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + regs->regs[4] = (long) error ? error : val; +} + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned long *args) +{ + args[0] = regs->orig_a0; + memcpy(&args[1], ®s->regs[5], 5 * sizeof(long)); +} + +static inline int syscall_get_arch(struct task_struct *task) +{ + return AUDIT_ARCH_LOONGARCH64; +} + +#endif /* __ASM_LOONGARCH_SYSCALL_H */ diff --git a/arch/loongarch/include/asm/thread_info.h b/arch/loongarch/include/asm/thread_info.h new file mode 100644 index 0000000000000000000000000000000000000000..fb5e3670ef0d168e22898f0cdef914dd7ae52780 --- /dev/null +++ b/arch/loongarch/include/asm/thread_info.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * thread_info.h: LoongArch low-level thread information + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_THREAD_INFO_H +#define _ASM_THREAD_INFO_H + +#ifdef __KERNEL__ + +#ifndef __ASSEMBLY__ + +#include + +/* + * low level task data that entry.S needs immediate access to + * - this struct should fit entirely inside of one cache line + * - this struct shares the supervisor stack pages + * - if the contents of this structure are changed, the assembly constants + * must also be changed + */ +struct thread_info { + struct task_struct *task; /* main task structure */ + unsigned long flags; /* low level flags */ + unsigned long tp_value; /* thread pointer */ + __u32 cpu; /* current CPU */ + int preempt_count; /* 0 => preemptable, <0 => BUG */ + struct pt_regs *regs; + long syscall; /* syscall number */ +}; + +/* + * macros/functions for gaining access to the thread information structure + */ +#define INIT_THREAD_INFO(tsk) \ +{ \ + .task = &tsk, \ + .flags = _TIF_FIXADE, \ + .cpu = 0, \ + .preempt_count = INIT_PREEMPT_COUNT, \ +} + +/* How to get the thread information struct from C. */ +register struct thread_info *__current_thread_info __asm__("$r2"); + +static inline struct thread_info *current_thread_info(void) +{ + return __current_thread_info; +} + +register unsigned long current_stack_pointer __asm__("$r3"); + +#endif /* !__ASSEMBLY__ */ + +/* thread information allocation */ +#define THREAD_SIZE SZ_16K +#define THREAD_MASK (THREAD_SIZE - 1UL) +#define THREAD_SIZE_ORDER ilog2(THREAD_SIZE / PAGE_SIZE) +/* + * thread information flags + * - these are process state flags that various assembly files may need to + * access + * - pending work-to-be-done flags are in LSW + * - other flags in MSW + */ +#define TIF_SIGPENDING 1 /* signal pending */ +#define TIF_NEED_RESCHED 2 /* rescheduling necessary */ +#define TIF_NOTIFY_RESUME 3 /* callback before returning to user */ +#define TIF_NOTIFY_SIGNAL 4 /* signal notifications exist */ +#define TIF_RESTORE_SIGMASK 5 /* restore signal mask in do_signal() */ +#define TIF_NOHZ 6 /* in adaptive nohz mode */ +#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ +#define TIF_SYSCALL_TRACE 8 /* syscall trace active */ +#define TIF_SYSCALL_TRACEPOINT 9 /* syscall tracepoint instrumentation */ +#define TIF_SECCOMP 10 /* secure computing */ +#define TIF_UPROBE 11 /* breakpointed or singlestepping */ +#define TIF_USEDFPU 12 /* FPU was used by this task this quantum (SMP) */ +#define TIF_USEDSIMD 13 /* SIMD has been used this quantum */ +#define TIF_MEMDIE 14 /* is terminating due to OOM killer */ +#define TIF_FIXADE 15 /* Fix address errors in software */ +#define TIF_LOGADE 16 /* Log address errors to syslog */ +#define TIF_32BIT_REGS 17 /* 32-bit general purpose registers */ +#define TIF_32BIT_ADDR 18 /* 32-bit address space */ +#define TIF_LOAD_WATCH 19 /* If set, load watch registers */ +#define TIF_SINGLESTEP 20 /* Single Step */ +#define TIF_LSX_CTX_LIVE 21 /* LSX context must be preserved */ +#define TIF_LASX_CTX_LIVE 22 /* LASX context must be preserved */ +#define TIF_PATCH_PENDING 23 /* pending live patching update */ +#define TIF_USEDLBT 24 /* LBT has been used */ +#define TIF_LBT_CTX_LIVE 25 /* LBT context */ + +#define _TIF_SIGPENDING (1< +#include +#include + +extern u64 cpu_clock_freq; +extern u64 const_clock_freq; + +extern void save_counter(void); +extern void sync_counter(void); + +static inline unsigned int calc_const_freq(void) +{ + unsigned int res; + unsigned int base_freq; + unsigned int cfm, cfd; + + res = read_cpucfg(LOONGARCH_CPUCFG2); + if (!(res & CPUCFG2_LLFTP)) + return 0; + + base_freq = read_cpucfg(LOONGARCH_CPUCFG4); + res = read_cpucfg(LOONGARCH_CPUCFG5); + cfm = res & 0xffff; + cfd = (res >> 16) & 0xffff; + + if (!base_freq || !cfm || !cfd) + return 0; + + return (base_freq * cfm / cfd); +} + +/* + * Initialize the calling CPU's timer interrupt as clockevent device + */ +extern int constant_clockevent_init(void); +extern int constant_clocksource_init(void); + +static inline void clockevent_set_clock(struct clock_event_device *cd, + unsigned int clock) +{ + clockevents_calc_mult_shift(cd, clock, 4); +} + +#endif /* _ASM_TIME_H */ diff --git a/arch/loongarch/include/asm/timex.h b/arch/loongarch/include/asm/timex.h new file mode 100644 index 0000000000000000000000000000000000000000..6d3aae7dbb448ac669c7b0f05d58dec9441b2240 --- /dev/null +++ b/arch/loongarch/include/asm/timex.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_TIMEX_H +#define _ASM_TIMEX_H + +#ifdef __KERNEL__ + +#include + +#include +#include +#include + +/* + * Standard way to access the cycle counter. + * Currently only used on SMP for scheduling. + * + * We know that all SMP capable CPUs have cycle counters. + */ + +typedef unsigned long cycles_t; + +#define get_cycles get_cycles + +static inline cycles_t get_cycles(void) +{ + return drdtime(); +} + +#endif /* __KERNEL__ */ + +#endif /* _ASM_TIMEX_H */ diff --git a/arch/loongarch/include/asm/tlb.h b/arch/loongarch/include/asm/tlb.h new file mode 100644 index 0000000000000000000000000000000000000000..fb0e984d1b28bd026fce6118849289c6df01f07a --- /dev/null +++ b/arch/loongarch/include/asm/tlb.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_TLB_H +#define __ASM_TLB_H + +#include +#include +#include + +/* + * TLB Invalidate Flush + */ +static inline void tlbclr(void) +{ + __asm__ __volatile__("tlbclr"); +} + +static inline void tlbflush(void) +{ + __asm__ __volatile__("tlbflush"); +} + +/* + * TLB R/W operations. + */ +static inline void tlb_probe(void) +{ + __asm__ __volatile__("tlbsrch"); +} + +static inline void tlb_read(void) +{ + __asm__ __volatile__("tlbrd"); +} + +static inline void tlb_write_indexed(void) +{ + __asm__ __volatile__("tlbwr"); +} + +static inline void tlb_write_random(void) +{ + __asm__ __volatile__("tlbfill"); +} + +enum invtlb_ops { + /* Invalid all tlb */ + INVTLB_ALL = 0x0, + /* Invalid current tlb */ + INVTLB_CURRENT_ALL = 0x1, + /* Invalid all global=1 lines in current tlb */ + INVTLB_CURRENT_GTRUE = 0x2, + /* Invalid all global=0 lines in current tlb */ + INVTLB_CURRENT_GFALSE = 0x3, + /* Invalid global=0 and matched asid lines in current tlb */ + INVTLB_GFALSE_AND_ASID = 0x4, + /* Invalid addr with global=0 and matched asid in current tlb */ + INVTLB_ADDR_GFALSE_AND_ASID = 0x5, + /* Invalid addr with global=1 or matched asid in current tlb */ + INVTLB_ADDR_GTRUE_OR_ASID = 0x6, + /* Invalid matched gid in guest tlb */ + INVGTLB_GID = 0x9, + /* Invalid global=1, matched gid in guest tlb */ + INVGTLB_GID_GTRUE = 0xa, + /* Invalid global=0, matched gid in guest tlb */ + INVGTLB_GID_GFALSE = 0xb, + /* Invalid global=0, matched gid and asid in guest tlb */ + INVGTLB_GID_GFALSE_ASID = 0xc, + /* Invalid global=0 , matched gid, asid and addr in guest tlb */ + INVGTLB_GID_GFALSE_ASID_ADDR = 0xd, + /* Invalid global=1 , matched gid, asid and addr in guest tlb */ + INVGTLB_GID_GTRUE_ASID_ADDR = 0xe, + /* Invalid all gid gva-->gpa guest tlb */ + INVGTLB_ALLGID_GVA_TO_GPA = 0x10, + /* Invalid all gid gpa-->hpa tlb */ + INVTLB_ALLGID_GPA_TO_HPA = 0x11, + /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */ + INVTLB_ALLGID = 0x12, + /* Invalid matched gid gva-->gpa guest tlb */ + INVGTLB_GID_GVA_TO_GPA = 0x13, + /* Invalid matched gid gpa-->hpa tlb */ + INVTLB_GID_GPA_TO_HPA = 0x14, + /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */ + INVTLB_GID_ALL = 0x15, + /* Invalid matched gid and addr gpa-->hpa tlb */ + INVTLB_GID_ADDR = 0x16, +}; + +static __always_inline void invtlb(u32 op, u32 info, u64 addr) +{ + __asm__ __volatile__( + "invtlb %0, %1, %2\n\t" + : + : "i"(op), "r"(info), "r"(addr) + : "memory" + ); +} + +static __always_inline void invtlb_addr(u32 op, u32 info, u64 addr) +{ + BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0); + __asm__ __volatile__( + "invtlb %0, $zero, %1\n\t" + : + : "i"(op), "r"(addr) + : "memory" + ); +} + +static __always_inline void invtlb_info(u32 op, u32 info, u64 addr) +{ + BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0); + __asm__ __volatile__( + "invtlb %0, %1, $zero\n\t" + : + : "i"(op), "r"(info) + : "memory" + ); +} + +static __always_inline void invtlb_all(u32 op, u32 info, u64 addr) +{ + BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0); + BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0); + __asm__ __volatile__( + "invtlb %0, $zero, $zero\n\t" + : + : "i"(op) + : "memory" + ); +} + +/* + * LoongArch doesn't need any special per-pte or per-vma handling, except + * we need to flush cache for area to be unmapped. + */ +#define tlb_start_vma(tlb, vma) \ + do { \ + if (!(tlb)->fullmm) \ + flush_cache_range(vma, vma->vm_start, vma->vm_end); \ + } while (0) +#define tlb_end_vma(tlb, vma) do { } while (0) +#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) + +static void tlb_flush(struct mmu_gather *tlb); + +#define tlb_flush tlb_flush +#include + +static inline void tlb_flush(struct mmu_gather *tlb) +{ + struct vm_area_struct vma; + + vma.vm_mm = tlb->mm; + vma.vm_flags = 0; + if (tlb->fullmm) { + flush_tlb_mm(tlb->mm); + return; + } + + flush_tlb_range(&vma, tlb->start, tlb->end); +} + +extern void handle_tlb_load(void); +extern void handle_tlb_store(void); +extern void handle_tlb_modify(void); +extern void handle_tlb_refill(void); +extern void handle_tlb_protect(void); +extern void handle_tlb_load_ptw(void); +extern void handle_tlb_store_ptw(void); +extern void handle_tlb_modify_ptw(void); + +extern void dump_tlb_all(void); +extern void dump_tlb_regs(void); + +#endif /* __ASM_TLB_H */ diff --git a/arch/loongarch/include/asm/tlbflush.h b/arch/loongarch/include/asm/tlbflush.h new file mode 100644 index 0000000000000000000000000000000000000000..784ef1750ed893643c57a59e4866b0c95ab78b0c --- /dev/null +++ b/arch/loongarch/include/asm/tlbflush.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_TLBFLUSH_H +#define __ASM_TLBFLUSH_H + +#include + +/* + * TLB flushing: + * + * - flush_tlb_all() flushes all processes TLB entries + * - flush_tlb_mm(mm) flushes the specified mm context TLB entries + * - flush_tlb_page(vma, vmaddr) flushes one page + * - flush_tlb_range(vma, start, end) flushes a range of pages + * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages + */ +extern void local_flush_tlb_all(void); +extern void local_flush_tlb_user(void); +extern void local_flush_tlb_kernel(void); +extern void local_flush_tlb_mm(struct mm_struct *mm); +extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); +extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); +extern void local_flush_tlb_one(unsigned long vaddr); + +#ifdef CONFIG_SMP + +extern void flush_tlb_all(void); +extern void flush_tlb_mm(struct mm_struct *); +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long, unsigned long); +extern void flush_tlb_kernel_range(unsigned long, unsigned long); +extern void flush_tlb_page(struct vm_area_struct *, unsigned long); +extern void flush_tlb_one(unsigned long vaddr); + +#else /* CONFIG_SMP */ + +#define flush_tlb_all() local_flush_tlb_all() +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end) +#define flush_tlb_kernel_range(vmaddr,end) local_flush_tlb_kernel_range(vmaddr, end) +#define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page) +#define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr) + +#endif /* CONFIG_SMP */ + +#endif /* __ASM_TLBFLUSH_H */ diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h new file mode 100644 index 0000000000000000000000000000000000000000..3c48c64e1ca15793b48e7df647fcdbd0f9bb1eb9 --- /dev/null +++ b/arch/loongarch/include/asm/topology.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_TOPOLOGY_H +#define __ASM_TOPOLOGY_H + +#include +#include + +#ifdef CONFIG_SMP +#define topology_physical_package_id(cpu) (cpu_data[cpu].package) +#define topology_core_id(cpu) (cpu_data[cpu].core) +#define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) +#define topology_sibling_cpumask(cpu) (&cpu_sibling_map[cpu]) +#endif + +#include + +static inline void arch_fix_phys_package_id(int num, u32 slot) { } +#endif /* __ASM_TOPOLOGY_H */ diff --git a/arch/loongarch/include/asm/trans_mips_syscalls.h b/arch/loongarch/include/asm/trans_mips_syscalls.h new file mode 100644 index 0000000000000000000000000000000000000000..c5332faaf26375a07a3611e0accd23d99476d623 --- /dev/null +++ b/arch/loongarch/include/asm/trans_mips_syscalls.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Authors: Hanlu Li + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _TRANS_MIPS_SYSCALLS_H +#define _TRANS_MIPS_SYSCALLS_H + +#define M_ENOMSG 35 +#define M_EIDRM 36 +#define M_ECHRNG 37 +#define M_EL2NSYNC 38 +#define M_EL3HLT 39 +#define M_EL3RST 40 +#define M_ELNRNG 41 +#define M_EUNATCH 42 +#define M_ENOCSI 43 +#define M_EL2HLT 44 +#define M_EDEADLK 45 +#define M_ENOLCK 46 +#define M_EBADE 50 +#define M_EBADR 51 +#define M_EXFULL 52 +#define M_ENOANO 53 +#define M_EBADRQC 54 +#define M_EBADSLT 55 +#define M_EBFONT 59 +#define M_ENOSTR 60 +#define M_ENODATA 61 +#define M_ETIME 62 +#define M_ENOSR 63 +#define M_ENONET 64 +#define M_ENOPKG 65 +#define M_EREMOTE 66 +#define M_ENOLINK 67 +#define M_EADV 68 +#define M_ESRMNT 69 +#define M_ECOMM 70 +#define M_EPROTO 71 +#define M_EDOTDOT 73 +#define M_EMULTIHOP 74 +#define M_EBADMSG 77 +#define M_ENAMETOOLONG 78 +#define M_EOVERFLOW 79 +#define M_ENOTUNIQ 80 +#define M_EBADFD 81 +#define M_EREMCHG 82 +#define M_ELIBACC 83 +#define M_ELIBBAD 84 +#define M_ELIBSCN 85 +#define M_ELIBMAX 86 +#define M_ELIBEXEC 87 +#define M_EILSEQ 88 +#define M_ENOSYS 89 +#define M_ELOOP 90 +#define M_ERESTART 91 +#define M_ESTRPIPE 92 +#define M_ENOTEMPTY 93 +#define M_EUSERS 94 +#define M_ENOTSOCK 95 +#define M_EDESTADDRREQ 96 +#define M_EMSGSIZE 97 +#define M_EPROTOTYPE 98 +#define M_ENOPROTOOPT 99 +#define M_EPROTONOSUPPORT 120 +#define M_ESOCKTNOSUPPORT 121 +#define M_EOPNOTSUPP 122 +#define M_EPFNOSUPPORT 123 +#define M_EAFNOSUPPORT 124 +#define M_EADDRINUSE 125 +#define M_EADDRNOTAVAIL 126 +#define M_ENETDOWN 127 +#define M_ENETUNREACH 128 +#define M_ENETRESET 129 +#define M_ECONNABORTED 130 +#define M_ECONNRESET 131 +#define M_ENOBUFS 132 +#define M_EISCONN 133 +#define M_ENOTCONN 134 +#define M_EUCLEAN 135 +#define M_ENOTNAM 137 +#define M_ENAVAIL 138 +#define M_EISNAM 139 +#define M_EREMOTEIO 140 +#define M_ESHUTDOWN 143 +#define M_ETOOMANYREFS 144 +#define M_ETIMEDOUT 145 +#define M_ECONNREFUSED 146 +#define M_EHOSTDOWN 147 +#define M_EHOSTUNREACH 148 +#define M_EALREADY 149 +#define M_EINPROGRESS 150 +#define M_ESTALE 151 +#define M_ECANCELED 158 +#define M_ENOMEDIUM 159 +#define M_EMEDIUMTYPE 160 +#define M_ENOKEY 161 +#define M_EKEYEXPIRED 162 +#define M_EKEYREVOKED 163 +#define M_EKEYREJECTED 164 +#define M_EOWNERDEAD 165 +#define M_ENOTRECOVERABLE 166 +#define M_ERFKILL 167 +#define M_EHWPOISON 168 +#define M_EDQUOT 1133 +#endif /* _TRANS_MIPS_SYSCALLS_H */ diff --git a/arch/loongarch/include/asm/trans_syscalls.h b/arch/loongarch/include/asm/trans_syscalls.h new file mode 100644 index 0000000000000000000000000000000000000000..f783c06b4517458d022eca51a7f703e64f83b84e --- /dev/null +++ b/arch/loongarch/include/asm/trans_syscalls.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Authors: Hanlu Li + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _TRANS_SYSCALLS_H_ +#define _TRANS_SYSCALLS_H_ +#include +#endif diff --git a/arch/loongarch/include/asm/types.h b/arch/loongarch/include/asm/types.h new file mode 100644 index 0000000000000000000000000000000000000000..18dd9c01cce470a3c9ff5ef9afaf674b692e448e --- /dev/null +++ b/arch/loongarch/include/asm/types.h @@ -0,0 +1,25 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994, 1995, 1996, 1999 by Ralf Baechle + * Copyright (C) 2008 Wind River Systems, + * written by Ralf Baechle + * Copyright (C) 1999 Silicon Graphics, Inc. + */ +#ifndef _ASM_TYPES_H +#define _ASM_TYPES_H + +#include +#include + +#ifdef __ASSEMBLY__ +#define _ULCAST_ +#define _U64CAST_ +#else +#define _ULCAST_ (unsigned long) +#define _U64CAST_ (u64) +#endif + +#endif /* _ASM_TYPES_H */ diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h new file mode 100644 index 0000000000000000000000000000000000000000..f2d837d03db96d601c66add7cda4302383de2551 --- /dev/null +++ b/arch/loongarch/include/asm/uaccess.h @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#ifndef _ASM_UACCESS_H +#define _ASM_UACCESS_H + +#include +#include +#include +#include +#include + +#ifdef CONFIG_64BIT + +extern u64 __ua_limit; + +#define __UA_LIMIT __ua_limit + +#define __UA_ADDR ".dword" +#define __UA_LA "la.abs" + +#endif /* CONFIG_64BIT */ + +/* + * Is a address valid? This does a straightforward calculation rather + * than tests. + * + * Address valid if: + * - "addr" doesn't have any high-bits set + * - AND "size" doesn't have any high-bits set + * - AND "addr+size" doesn't have any high-bits set + * - OR we are in kernel mode. + * + * __ua_size() is a trick to avoid runtime checking of positive constant + * sizes; for those we already know at compile time that the size is ok. + */ +#define __ua_size(size) \ + ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size)) + +/* + * access_ok: - Checks if a user space pointer is valid + * @addr: User space pointer to start of block to check + * @size: Size of block to check + * + * Context: User context only. This function may sleep if pagefaults are + * enabled. + * + * Checks if a pointer to a block of memory in user space is valid. + * + * Returns true (nonzero) if the memory block may be valid, false (zero) + * if it is definitely invalid. + * + * Note that, depending on architecture, this function probably just + * checks that the pointer is in the user space range - after calling + * this function, memory access functions may still return -EFAULT. + */ +static inline int __access_ok(const void __user *p, unsigned long size) +{ + unsigned long addr = (unsigned long)p; + unsigned long end = addr + size - !!size; + + return (__UA_LIMIT & (addr | end | __ua_size(size))) == 0; +} + +#define access_ok(addr, size) \ + likely(__access_ok((addr), (size))) + +/* + * get_user: - Get a simple variable from user space. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep if pagefaults are + * enabled. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define get_user(x,ptr) \ +({ \ + const __typeof__(*(ptr)) __user *__p = (ptr); \ + \ + might_fault(); \ + access_ok(__p, sizeof(*__p)) ? __get_user((x), __p) : \ + ((x) = 0, -EFAULT); \ +}) + +/* + * put_user: - Write a simple value into user space. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep if pagefaults are + * enabled. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Returns zero on success, or -EFAULT on error. + */ +#define put_user(x,ptr) \ +({ \ + __typeof__(*(ptr)) __user *__p = (ptr); \ + \ + might_fault(); \ + access_ok(__p, sizeof(*__p)) ? __put_user((x), __p) : -EFAULT; \ +}) + +/* + * __get_user: - Get a simple variable from user space, with less checking. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep if pagefaults are + * enabled. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define __get_user(x,ptr) \ +({ \ + int __gu_err = 0; \ + \ + __chk_user_ptr(ptr); \ + __get_user_common((x), sizeof(*(ptr)), ptr); \ + __gu_err; \ +}) + +/* + * __put_user: - Write a simple value into user space, with less checking. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep if pagefaults are + * enabled. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + */ +#define __put_user(x,ptr) \ +({ \ + int __pu_err = 0; \ + __typeof__(*(ptr)) __pu_val; \ + \ + __pu_val = (x); \ + __chk_user_ptr(ptr); \ + __put_user_common(ptr, sizeof(*(ptr))); \ + __pu_err; \ +}) + +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + +#ifdef CONFIG_32BIT +#define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr) +#endif +#ifdef CONFIG_64BIT +#define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr) +#endif + +#define __get_user_common(val, size, ptr) \ +do { \ + switch (size) { \ + case 1: __get_data_asm(val, "ld.b", ptr); break; \ + case 2: __get_data_asm(val, "ld.h", ptr); break; \ + case 4: __get_data_asm(val, "ld.w", ptr); break; \ + case 8: __GET_DW(val, "ld.d", ptr); break; \ + default: BUILD_BUG(); break; \ + } \ +} while (0) + +#define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr) + +#define __get_data_asm(val, insn, ptr) \ +{ \ + long __gu_tmp; \ + \ + __asm__ __volatile__( \ + "1: " insn " %1, %2 \n" \ + "2: \n" \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %0, %1) \ + : "+r" (__gu_err), "=r" (__gu_tmp) \ + : "m" (__m(ptr))); \ + \ + (val) = (__typeof__(*(ptr))) __gu_tmp; \ +} + +/* + * Get a long long 64 using 32 bit registers. + */ +#define __get_data_asm_ll32(val, insn, ptr) \ +{ \ + union { \ + unsigned long long l; \ + __typeof__(*(addr)) t; \ + } __gu_tmp; \ + \ + __asm__ __volatile__( \ + "1: ld.w %1, (%2) \n" \ + "2: ld.w %D1, 4(%2) \n" \ + "3: \n" \ + " .section .fixup,\"ax\" \n" \ + "4: li.w %0, %3 \n" \ + " slli.d %1, $r0, 0 \n" \ + " slli.d %D1, $r0, 0 \n" \ + " b 3b \n" \ + " .previous \n" \ + " .section __ex_table,\"a\" \n" \ + " " __UA_ADDR " 1b, 4b \n" \ + " " __UA_ADDR " 2b, 4b \n" \ + " .previous \n" \ + : "+r" (__gu_err), "=&r" (__gu_tmp.l) \ + : "r" (ptr), "i" (-EFAULT)); \ + \ + (val) = __gu_tmp.t; \ +} + +#ifdef CONFIG_32BIT +#define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr) +#endif +#ifdef CONFIG_64BIT +#define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr) +#endif + +#define __put_user_common(ptr, size) \ +do { \ + switch (size) { \ + case 1: __put_data_asm("st.b", ptr); break; \ + case 2: __put_data_asm("st.h", ptr); break; \ + case 4: __put_data_asm("st.w", ptr); break; \ + case 8: __PUT_DW("st.d", ptr); break; \ + default: BUILD_BUG(); break; \ + } \ +} while (0) + +#define __put_kernel_common(ptr, size) __put_user_common(ptr, size) + +#define __put_data_asm(insn, ptr) \ +{ \ + __asm__ __volatile__( \ + "1: " insn " %z2, %1 # __put_user_asm\n" \ + "2: \n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %0) \ + : "+r" (__pu_err), "=m" (__m(ptr)) \ + : "Jr" (__pu_val)); \ +} + +#define __put_data_asm_ll32(insn, ptr) \ +{ \ + __asm__ __volatile__( \ + "1: st.w %1, (%2) # __put_user_asm_ll32 \n" \ + "2: st.w %D1, 4(%2) \n" \ + "3: \n" \ + " .section .fixup,\"ax\" \n" \ + "4: li.w %0, %3 \n" \ + " b 3b \n" \ + " .previous \n" \ + " .section __ex_table,\"a\" \n" \ + " " __UA_ADDR " 1b, 4b \n" \ + " " __UA_ADDR " 2b, 4b \n" \ + " .previous" \ + : "+r" (__pu_err) \ + : "r" (__pu_val), "r" (ptr), "i" (-EFAULT)); \ +} + +#define HAVE_GET_KERNEL_NOFAULT + +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + int __gu_err = 0; \ + \ + __get_kernel_common(*((type *)(dst)), sizeof(type), \ + (__force type *)(src)); \ + if (unlikely(__gu_err)) \ + goto err_label; \ +} while (0) + +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + type __pu_val; \ + int __pu_err = 0; \ + \ + __pu_val = *(__force type *)(src); \ + __put_kernel_common(((type *)(dst)), sizeof(type)); \ + if (unlikely(__pu_err)) \ + goto err_label; \ +} while (0) + +extern unsigned long __copy_user(void *to, const void *from, __kernel_size_t n); + +static inline unsigned long __must_check +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return __copy_user(to, (__force const void *)from, n); +} + +static inline unsigned long __must_check +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return __copy_user((__force void *)to, from, n); +} + +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER + +/* + * __clear_user: - Zero a block of memory in user space, with less checking. + * @addr: Destination address, in user space. + * @to: Number of bytes to zero. + * + * Zero a block of memory in user space. Caller must check + * the specified block with access_ok() before calling this function. + * + * Returns number of bytes that could not be cleared. + * On success, this will be zero. + */ +extern unsigned long __clear_user(void __user *addr, __kernel_size_t size); + +#define clear_user(addr,n) \ +({ \ + void __user * __cl_addr = (addr); \ + unsigned long __cl_size = (n); \ + if (__cl_size && access_ok(__cl_addr, __cl_size)) \ + __cl_size = __clear_user(__cl_addr, __cl_size); \ + __cl_size; \ +}) + +extern long strncpy_from_user(char *dest, const char __user *src, long count); +extern long strnlen_user(const char __user *str, long n); + +#endif /* _ASM_UACCESS_H */ diff --git a/arch/loongarch/include/asm/unistd.h b/arch/loongarch/include/asm/unistd.h new file mode 100644 index 0000000000000000000000000000000000000000..db63d8a2905b6eb249e649e08e65a50f5fe30b51 --- /dev/null +++ b/arch/loongarch/include/asm/unistd.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ + +#include + +#define NR_syscalls (__NR_syscalls) + +/* for binary translation */ +#if defined(CONFIG_CPU_HAS_LBT) +#define __ARCH_WANT_SYS_ALARM +#define __ARCH_WANT_SYS_GETPGRP + +#define __NR_MIPS64_Linux 5000 +#define __NR_MIPS64_Linux_syscalls 329 +#define __NR_i386_Linux_syscalls 386 + +#define TRANS_MIPS_N64 0x010000 +#define TRANS_I386 0x100000 +#define TRANS_X64 0x110000 +#define TRANS_ARCH_MASK 0xffff0000 +#define SYS_NUM_MASK 0xffff +#endif diff --git a/arch/loongarch/include/asm/unwind.h b/arch/loongarch/include/asm/unwind.h new file mode 100644 index 0000000000000000000000000000000000000000..9833dece49f989a0e532e88ab169abe79b5ff359 --- /dev/null +++ b/arch/loongarch/include/asm/unwind.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Most of this ideas comes from x86. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_UNWIND_H +#define _ASM_UNWIND_H + +#include +#include +#include + +#include +#include + +struct unwind_state { + struct stack_info stack_info; + struct task_struct *task; + int graph_idx; +#if defined(CONFIG_UNWINDER_PROLOGUE) + unsigned long sp, pc, ra; + bool enable; + bool first; +#elif defined(CONFIG_UNWINDER_ORC) + unsigned long sp, pc, fp, ra; +#else /* CONFIG_UNWINDER_GUESS */ + unsigned long sp, pc; + bool first; +#endif + bool error, is_ftrace; +}; + +void unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs); +bool unwind_next_frame(struct unwind_state *state); +unsigned long unwind_get_return_address(struct unwind_state *state); + +static inline bool unwind_done(struct unwind_state *state) +{ + return state->stack_info.type == STACK_TYPE_UNKNOWN; +} + +static inline bool unwind_error(struct unwind_state *state) +{ + return state->error; +} + +#ifdef CONFIG_UNWINDER_ORC +void unwind_init(void); +void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size); +#else +static inline void unwind_init(void) {} +static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} +#endif /* CONFIG_UNWINDER_ORC */ + +#define GRAPH_FAKE_OFFSET (sizeof(struct pt_regs) - offsetof(struct pt_regs, regs[1])) + +static inline unsigned long unwind_graph_addr(struct unwind_state *state, + unsigned long pc, unsigned long cfa) +{ + return ftrace_graph_ret_addr(state->task, &state->graph_idx, + pc, (unsigned long *)(cfa - GRAPH_FAKE_OFFSET)); +} +#endif /* _ASM_UNWIND_H */ diff --git a/arch/loongarch/include/asm/unwind_hints.h b/arch/loongarch/include/asm/unwind_hints.h new file mode 100644 index 0000000000000000000000000000000000000000..51fbddff045823cd7bf3a8052c83f80652d7414d --- /dev/null +++ b/arch/loongarch/include/asm/unwind_hints.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Most of this ideas comes from x86. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_UNWIND_HINTS_H +#define _ASM_UNWIND_HINTS_H + +#include +#include "orc_types.h" + +#ifdef __ASSEMBLY__ + +.macro UNWIND_HINT_EMPTY + UNWIND_HINT sp_reg=ORC_REG_UNDEFINED type=UNWIND_HINT_TYPE_CALL end=1 +.endm + +.macro UNWIND_HINT_REGS base=ORC_REG_SP offset=0 + UNWIND_HINT sp_reg=\base sp_offset=\offset type=UNWIND_HINT_TYPE_REGS +.endm + +.macro UNWIND_HINT_FUNC offset=0 + UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=\offset type=UNWIND_HINT_TYPE_FUNC +.endm + +.macro NOT_SIBLING_CALL_HINT +876: .pushsection .discard.not_sibling_call + .long 876b - . + .popsection +.endm + +#else /* !__ASSEMBLY__ */ + +#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE, 0) + +#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE, 0) + +#define NOT_SIBLING_CALL_HINT \ + "876:\n\t" \ + ".pushsection .discard.not_sibling_call\n\t" \ + ".long 876b - .\n\t" \ + ".popsection\n\t" + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_UNWIND_HINTS_H */ diff --git a/arch/loongarch/include/asm/uprobes.h b/arch/loongarch/include/asm/uprobes.h new file mode 100644 index 0000000000000000000000000000000000000000..3760f9033e5bc9fe1749ed1f9211994841e4ad38 --- /dev/null +++ b/arch/loongarch/include/asm/uprobes.h @@ -0,0 +1,40 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#ifndef __ASM_UPROBES_H +#define __ASM_UPROBES_H + +#include +#include + +#include +#include + +/* + * We want this to be defined as union loongarch_instruction but that makes the + * generic code blow up. + */ +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 8 +#define UPROBE_XOL_SLOT_BYTES 128 /* Max. cache line size */ + +#define UPROBE_BRK_UPROBE 0x002a000c /* break 12 */ +#define UPROBE_BRK_UPROBE_XOL 0x002a000d /* break 13 */ + +#define UPROBE_SWBP_INSN UPROBE_BRK_UPROBE +#define UPROBE_SWBP_INSN_SIZE 4 + +struct arch_uprobe { + unsigned long resume_era; + u32 insn[2]; + u32 ixol[2]; +}; + +struct arch_uprobe_task { + unsigned long saved_trap_nr; +}; + +#endif /* __ASM_UPROBES_H */ diff --git a/arch/loongarch/include/asm/vdso.h b/arch/loongarch/include/asm/vdso.h new file mode 100644 index 0000000000000000000000000000000000000000..f81602a9a069c41c9c2e947bff320fe18fc4e5c9 --- /dev/null +++ b/arch/loongarch/include/asm/vdso.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#ifndef __ASM_VDSO_H +#define __ASM_VDSO_H + +#include +#include +#include + +#include + +/* + * struct loongarch_vdso_info - Details of a VDSO image. + * @vdso: Pointer to VDSO image (page-aligned). + * @size: Size of the VDSO image (page-aligned). + * @off_rt_sigreturn: Offset of the rt_sigreturn() trampoline. + * @code_mapping: Special mapping structure for vdso code. + * @code_mapping: Special mapping structure for vdso data. + * + * This structure contains details of a VDSO image, including the image data + * and offsets of certain symbols required by the kernel. It is generated as + * part of the VDSO build process, aside from the mapping page array, which is + * populated at runtime. + */ +struct loongarch_vdso_info { + void *vdso; + unsigned long size; + unsigned long offset_sigreturn; + struct vm_special_mapping code_mapping; + struct vm_special_mapping data_mapping; +}; + +extern struct loongarch_vdso_info vdso_info; + +#endif /* __ASM_VDSO_H */ diff --git a/arch/loongarch/include/asm/vdso/clocksource.h b/arch/loongarch/include/asm/vdso/clocksource.h new file mode 100644 index 0000000000000000000000000000000000000000..13cd580d406d2f91f7522eb250245a8011067268 --- /dev/null +++ b/arch/loongarch/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_CPU + +#endif /* __ASM_VDSOCLOCKSOURCE_H */ diff --git a/arch/loongarch/include/asm/vdso/gettimeofday.h b/arch/loongarch/include/asm/vdso/gettimeofday.h new file mode 100644 index 0000000000000000000000000000000000000000..e7a10d62d077f760da44c57d10107a909cdcbecb --- /dev/null +++ b/arch/loongarch/include/asm/vdso/gettimeofday.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline long gettimeofday_fallback( + struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long nr asm("a7") = __NR_gettimeofday; + register long ret asm("a0"); + + asm volatile( + " syscall 0\n" + : "+r" (ret) + : "r" (nr), "r" (tv), "r" (tz) + : "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", + "$t8", "memory"); + + return ret; +} + +static __always_inline long clock_gettime_fallback( + clockid_t _clkid, + struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long nr asm("a7") = __NR_clock_gettime; + register long ret asm("a0"); + + asm volatile( + " syscall 0\n" + : "+r" (ret) + : "r" (nr), "r" (clkid), "r" (ts) + : "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", + "$t8", "memory"); + + return ret; +} + +static __always_inline int clock_getres_fallback( + clockid_t _clkid, + struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long nr asm("a7") = __NR_clock_getres; + register long ret asm("a0"); + + asm volatile( + " syscall 0\n" + : "+r" (ret) + : "r" (nr), "r" (clkid), "r" (ts) + : "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", + "$t8", "memory"); + + return ret; +} + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ + uint64_t count; + + __asm__ __volatile__( + " rdtime.d %0, $zero\n" + : "=r" (count)); + + return count; +} + +static inline bool loongarch_vdso_hres_capable(void) +{ + return true; +} +#define __arch_vdso_hres_capable loongarch_vdso_hres_capable + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return (const struct vdso_data *)get_vdso_data(); +} + +#ifdef CONFIG_TIME_NS +static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) +{ + return (const struct vdso_data *)(get_vdso_data() + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE); +} +#endif +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/loongarch/include/asm/vdso/processor.h b/arch/loongarch/include/asm/vdso/processor.h new file mode 100644 index 0000000000000000000000000000000000000000..99aa47ca40e45569632f281b127540dd203669cb --- /dev/null +++ b/arch/loongarch/include/asm/vdso/processor.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#define cpu_relax() barrier() + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/loongarch/include/asm/vdso/vdso.h b/arch/loongarch/include/asm/vdso/vdso.h new file mode 100644 index 0000000000000000000000000000000000000000..567be88d0fa505cd3b4bc6d4563330552248b7dc --- /dev/null +++ b/arch/loongarch/include/asm/vdso/vdso.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Huacai Chen + */ + +#ifndef __ASSEMBLY__ + +#include +#include +#include + +struct vdso_pcpu_data { + u32 node; +} ____cacheline_aligned_in_smp; + +struct loongarch_vdso_data { + struct vdso_pcpu_data pdata[NR_CPUS]; +}; + +/* + * The layout of vvar: + * + * high + * +---------------------+--------------------------+ + * | loongarch vdso data | LOONGARCH_VDSO_DATA_SIZE | + * +---------------------+--------------------------+ + * | time-ns vdso data | PAGE_SIZE | + * +---------------------+--------------------------+ + * | generic vdso data | PAGE_SIZE | + * +---------------------+--------------------------+ + * low + */ +#define LOONGARCH_VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data)) +#define LOONGARCH_VDSO_DATA_PAGES (LOONGARCH_VDSO_DATA_SIZE >> PAGE_SHIFT) + +enum vvar_pages { + VVAR_GENERIC_PAGE_OFFSET, + VVAR_TIMENS_PAGE_OFFSET, + VVAR_LOONGARCH_PAGES_START, + VVAR_LOONGARCH_PAGES_END = VVAR_LOONGARCH_PAGES_START + LOONGARCH_VDSO_DATA_PAGES - 1, + VVAR_NR_PAGES, +}; + +#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) + +static inline unsigned long get_vdso_base(void) +{ + unsigned long addr; + + __asm__( + " la.pcrel %0, _start \n" + : "=r" (addr) + : + :); + + return addr; +} + +static inline unsigned long get_vdso_data(void) +{ + return get_vdso_base() - VVAR_SIZE; +} + +#endif /* __ASSEMBLY__ */ diff --git a/arch/loongarch/include/asm/vdso/vsyscall.h b/arch/loongarch/include/asm/vdso/vsyscall.h new file mode 100644 index 0000000000000000000000000000000000000000..5de615383a22f4849beeedd8eb30c6824ec65a85 --- /dev/null +++ b/arch/loongarch/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include +#include + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline +struct vdso_data *__loongarch_get_k_vdso_data(void) +{ + return vdso_data; +} +#define __arch_get_k_vdso_data __loongarch_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/loongarch/include/asm/vermagic.h b/arch/loongarch/include/asm/vermagic.h new file mode 100644 index 0000000000000000000000000000000000000000..aacbce545a796f6c2c6a3d448f0f8bdda52e7fcd --- /dev/null +++ b/arch/loongarch/include/asm/vermagic.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _ASM_VERMAGIC_H +#define _ASM_VERMAGIC_H + +#define MODULE_PROC_FAMILY "LOONGARCH " + +#ifdef CONFIG_32BIT +#define MODULE_KERNEL_TYPE "32BIT " +#elif defined CONFIG_64BIT +#define MODULE_KERNEL_TYPE "64BIT " +#endif + +#define MODULE_ARCH_VERMAGIC \ + MODULE_PROC_FAMILY MODULE_KERNEL_TYPE + +#endif /* _ASM_VERMAGIC_H */ diff --git a/arch/loongarch/include/asm/vmalloc.h b/arch/loongarch/include/asm/vmalloc.h new file mode 100644 index 0000000000000000000000000000000000000000..965a0d41ac2d50064ad275e32f0d1a02d142109f --- /dev/null +++ b/arch/loongarch/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_LOONGARCH_VMALLOC_H +#define _ASM_LOONGARCH_VMALLOC_H + +#endif /* _ASM_LOONGARCH_VMALLOC_H */ diff --git a/arch/loongarch/include/asm/watch.h b/arch/loongarch/include/asm/watch.h new file mode 100644 index 0000000000000000000000000000000000000000..ae68e441edede2f2b0fa9c662d09fa7c8e173f63 --- /dev/null +++ b/arch/loongarch/include/asm/watch.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Chong Qiao + * Huacai Chen + * + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#ifndef _ASM_WATCH_H +#define _ASM_WATCH_H + +#include + +#include + +unsigned long watch_csrrd(unsigned int reg); +void watch_csrwr(unsigned long val, unsigned int reg); + +void loongarch_probe_watch_registers(struct cpuinfo_loongarch *c); +void loongarch_clear_watch_registers(void); +void loongarch_install_watch_registers(struct task_struct *t); +void loongarch_clear_prev_watch_registers(struct task_struct *prev); +void loongarch_install_next_watch_registers(struct task_struct *next); +void loongarch_read_watch_registers(struct pt_regs *regs); +void loongarch_update_watch_registers(struct task_struct *t); + +#ifdef CONFIG_HARDWARE_WATCHPOINTS +#define __process_watch(prev, next) do { \ + if (test_bit(TIF_LOAD_WATCH, &task_thread_info(prev)->flags) \ + || test_bit(TIF_SINGLESTEP, &task_thread_info(prev)->flags)) \ + loongarch_clear_prev_watch_registers(prev); \ + if (test_bit(TIF_LOAD_WATCH, &task_thread_info(prev)->flags) \ + || test_bit(TIF_SINGLESTEP, &task_thread_info(prev)->flags)) \ + loongarch_install_next_watch_registers(next); \ +} while (0) +#else +#define __process_watch(prev, next) do {} while (0) +#endif + +#endif /* _ASM_WATCH_H */ diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h new file mode 100644 index 0000000000000000000000000000000000000000..12467fffee46875b444b500532f98e001b59eb8b --- /dev/null +++ b/arch/loongarch/include/asm/xor.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2023 WANG Xuerui + */ +#ifndef _ASM_LOONGARCH_XOR_H +#define _ASM_LOONGARCH_XOR_H + +#include +#include + +#ifdef CONFIG_CPU_HAS_LSX +static struct xor_block_template xor_block_lsx = { + .name = "lsx", + .do_2 = xor_lsx_2, + .do_3 = xor_lsx_3, + .do_4 = xor_lsx_4, + .do_5 = xor_lsx_5, +}; + +#define XOR_SPEED_LSX() \ + do { \ + if (cpu_has_lsx) \ + xor_speed(&xor_block_lsx); \ + } while (0) +#else /* CONFIG_CPU_HAS_LSX */ +#define XOR_SPEED_LSX() +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +static struct xor_block_template xor_block_lasx = { + .name = "lasx", + .do_2 = xor_lasx_2, + .do_3 = xor_lasx_3, + .do_4 = xor_lasx_4, + .do_5 = xor_lasx_5, +}; + +#define XOR_SPEED_LASX() \ + do { \ + if (cpu_has_lasx) \ + xor_speed(&xor_block_lasx); \ + } while (0) +#else /* CONFIG_CPU_HAS_LASX */ +#define XOR_SPEED_LASX() +#endif /* CONFIG_CPU_HAS_LASX */ + +/* + * For grins, also test the generic routines. + * + * More importantly: it cannot be ruled out at this point of time, that some + * future (maybe reduced) models could run the vector algorithms slower than + * the scalar ones, maybe for errata or micro-op reasons. It may be + * appropriate to revisit this after one or two more uarch generations. + */ +#include + +#undef XOR_TRY_TEMPLATES +#define XOR_TRY_TEMPLATES \ +do { \ + xor_speed(&xor_block_8regs); \ + xor_speed(&xor_block_8regs_p); \ + xor_speed(&xor_block_32regs); \ + xor_speed(&xor_block_32regs_p); \ + XOR_SPEED_LSX(); \ + XOR_SPEED_LASX(); \ +} while (0) + +#endif /* _ASM_LOONGARCH_XOR_H */ diff --git a/arch/loongarch/include/asm/xor_simd.h b/arch/loongarch/include/asm/xor_simd.h new file mode 100644 index 0000000000000000000000000000000000000000..809159293da2cf439ac2d0bd27d2f238b2edd8d3 --- /dev/null +++ b/arch/loongarch/include/asm/xor_simd.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2023 WANG Xuerui + */ +#ifndef _ASM_LOONGARCH_XOR_SIMD_H +#define _ASM_LOONGARCH_XOR_SIMD_H + +#ifdef CONFIG_CPU_HAS_LSX +void xor_lsx_2(unsigned long bytes, unsigned long *p1, + unsigned long *p2); +void xor_lsx_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3); +void xor_lsx_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4); +void xor_lsx_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5); +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +void xor_lasx_2(unsigned long bytes, unsigned long *p1, + unsigned long *p2); +void xor_lasx_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3); +void xor_lasx_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4); +void xor_lasx_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5); +#endif /* CONFIG_CPU_HAS_LASX */ + +#endif /* _ASM_LOONGARCH_XOR_SIMD_H */ diff --git a/arch/loongarch/include/uapi/asm/Kbuild b/arch/loongarch/include/uapi/asm/Kbuild new file mode 100644 index 0000000000000000000000000000000000000000..f66554cd5c4518cf919abf321ce9137e8b2be59d --- /dev/null +++ b/arch/loongarch/include/uapi/asm/Kbuild @@ -0,0 +1 @@ +# SPDX-License-Identifier: GPL-2.0 diff --git a/arch/loongarch/include/uapi/asm/auxvec.h b/arch/loongarch/include/uapi/asm/auxvec.h new file mode 100644 index 0000000000000000000000000000000000000000..bb0c71b4d64c49762438b01e5496791032769652 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/auxvec.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ + +#ifndef __ASM_AUXVEC_H +#define __ASM_AUXVEC_H + +/* Location of VDSO image. */ +#define AT_SYSINFO_EHDR 33 + +#define AT_VECTOR_SIZE_ARCH 1 /* entries in ARCH_DLINFO */ + +#endif /* __ASM_AUXVEC_H */ diff --git a/arch/loongarch/include/uapi/asm/bitfield.h b/arch/loongarch/include/uapi/asm/bitfield.h new file mode 100644 index 0000000000000000000000000000000000000000..1bdadee88617955b1b638ad7a67aa739c7377a40 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/bitfield.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ +#ifndef __UAPI_ASM_BITFIELD_H +#define __UAPI_ASM_BITFIELD_H + +/* + * * Damn ... bitfields depend from byteorder :-( + * */ +#define __BITFIELD_FIELD(field, more) \ + more \ + field; + +#endif /* __UAPI_ASM_BITFIELD_H */ diff --git a/arch/loongarch/include/uapi/asm/bitsperlong.h b/arch/loongarch/include/uapi/asm/bitsperlong.h new file mode 100644 index 0000000000000000000000000000000000000000..00b4ba1e5cdf032f81aff9f728dcffea68571880 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/bitsperlong.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_LOONGARCH_BITSPERLONG_H +#define __ASM_LOONGARCH_BITSPERLONG_H + +#define __BITS_PER_LONG (__SIZEOF_LONG__ * 8) + +#include + +#endif /* __ASM_LOONGARCH_BITSPERLONG_H */ diff --git a/arch/loongarch/include/uapi/asm/bpf_perf_event.h b/arch/loongarch/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 0000000000000000000000000000000000000000..eb6e2fd2a1f01eaddfd0dfb31bdc868e1b12fb81 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ +#define _UAPI__ASM_BPF_PERF_EVENT_H__ + +#include + +typedef struct user_pt_regs bpf_user_pt_regs_t; + +#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ diff --git a/arch/loongarch/include/uapi/asm/break.h b/arch/loongarch/include/uapi/asm/break.h new file mode 100644 index 0000000000000000000000000000000000000000..414b0f49e8eaaad5c3e21464e0ae0b3cfddda4d1 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/break.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1999 Silicon Graphics, Inc. + */ +#ifndef __UAPI_ASM_BREAK_H +#define __UAPI_ASM_BREAK_H + +#define BRK_DEFAULT 0 /* Used as default */ +#define BRK_BUG 1 /* Used by BUG() */ +#define BRK_KDB 2 /* Used in KDB_ENTER() */ +#define BRK_MATHEMU 3 /* Used by FPU emulator */ +#define BRK_USERBP 4 /* User bp (used by debuggers) */ +#define BRK_SSTEPBP 5 /* User bp (used by debuggers) */ +#define BRK_OVERFLOW 6 /* Overflow check */ +#define BRK_DIVZERO 7 /* Divide by zero check */ +#define BRK_RANGE 8 /* Range error check */ +#define BRK_MULOVFL 9 /* Multiply overflow */ +#define BRK_KPROBE_BP 10 /* Kprobe break */ +#define BRK_KPROBE_SSTEPBP 11 /* Kprobe single step break */ +#define BRK_UPROBE_BP 12 /* See */ +#define BRK_UPROBE_XOLBP 13 /* See */ + +#endif /* __UAPI_ASM_BREAK_H */ diff --git a/arch/loongarch/include/uapi/asm/byteorder.h b/arch/loongarch/include/uapi/asm/byteorder.h new file mode 100644 index 0000000000000000000000000000000000000000..bbb85c62740cd3812c0ceceefecd7416615e0451 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/byteorder.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ +#ifndef _ASM_BYTEORDER_H +#define _ASM_BYTEORDER_H + +#include + +#endif /* _ASM_BYTEORDER_H */ diff --git a/arch/loongarch/include/uapi/asm/hwcap.h b/arch/loongarch/include/uapi/asm/hwcap.h new file mode 100644 index 0000000000000000000000000000000000000000..6955a7cb2c65d1b575dd9ef9af3b1616fdd3cbab --- /dev/null +++ b/arch/loongarch/include/uapi/asm/hwcap.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_HWCAP_H +#define _UAPI_ASM_HWCAP_H + +/* HWCAP flags */ +#define HWCAP_LOONGARCH_CPUCFG (1 << 0) +#define HWCAP_LOONGARCH_LAM (1 << 1) +#define HWCAP_LOONGARCH_UAL (1 << 2) +#define HWCAP_LOONGARCH_FPU (1 << 3) +#define HWCAP_LOONGARCH_LSX (1 << 4) +#define HWCAP_LOONGARCH_LASX (1 << 5) +#define HWCAP_LOONGARCH_CRC32 (1 << 6) +#define HWCAP_LOONGARCH_COMPLEX (1 << 7) +#define HWCAP_LOONGARCH_CRYPTO (1 << 8) +#define HWCAP_LOONGARCH_LVZ (1 << 9) +#define HWCAP_LOONGARCH_LBT_X86 (1 << 10) +#define HWCAP_LOONGARCH_LBT_ARM (1 << 11) +#define HWCAP_LOONGARCH_LBT_MIPS (1 << 12) +#define HWCAP_LOONGARCH_PTW (1 << 13) + +#endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/loongarch/include/uapi/asm/inst.h b/arch/loongarch/include/uapi/asm/inst.h new file mode 100644 index 0000000000000000000000000000000000000000..de8c554bb7aa174010fa4bd0216c604b3386fbef --- /dev/null +++ b/arch/loongarch/include/uapi/asm/inst.h @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Format of an instruction in memory. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#ifndef _UAPI_ASM_INST_H +#define _UAPI_ASM_INST_H + +#include + +enum reg0_op { + tlbclr_op = 0x19208, gtlbclr_op=0x19208, + tlbflush_op = 0x19209, gtlbflush_op=0x19209, + tlbsrch_op = 0x1920a, gtlbsrch_op=0x1920a, + tlbrd_op = 0x1920b, gtlbrd_op=0x1920b, + tlbwr_op = 0x1920c, gtlbwr_op=0x1920c, + tlbfill_op = 0x1920d, gtlbfill_op=0x1920d, + ertn_op = 0x1920e, +}; + +enum reg0i15_op { + break_op = 0x54, dbcl_op, syscall_op, hypcall_op, + idle_op = 0xc91, dbar_op = 0x70e4, ibar_op, +}; + +enum reg0i26_op { + b_op = 0x14, bl_op, +}; + +enum reg1i20_op { + lu12iw_op = 0xa, lu32id_op, pcaddi_op, pcalau12i_op, + pcaddu12i_op, pcaddu18i_op, +}; + +enum reg1i21_op { + beqz_op = 0x10, bnez_op, bceqz_op, bcnez_op=0x12, jiscr0_op=0x12, jiscr1_op=0x12, +}; + +enum reg2_op { + gr2scr_op = 0x2, scr2gr_op, clow_op, + clzw_op, ctow_op, ctzw_op, clod_op, + clzd_op, ctod_op, ctzd_op, revb2h_op, + revb4h_op, revb2w_op, revbd_op, revh2w_op, + revhd_op, bitrev4b_op, bitrev8b_op, bitrevw_op, + bitrevd_op, extwh_op, extwb_op, rdtimelw_op, + rdtimehw_op, rdtimed_op, cpucfg_op, + iocsrrdb_op = 0x19200, iocsrrdh_op, iocsrrdw_op, iocsrrdd_op, + iocsrwrb_op, iocsrwrh_op, iocsrwrw_op, iocsrwrd_op, + movgr2fcsr_op = 0x4530, movfcsr2gr_op = 0x4532, + movgr2cf_op = 0x4536, movcf2gr_op = 0x4537, +}; + +enum reg2ui3_op { + rotrib_op = 0x261, rcrib_op = 0x281, +}; + +enum reg2ui4_op { + rotrih_op = 0x131, rcrih_op = 0x141, +}; + +enum reg2ui5_op { + slliw_op = 0x81, srliw_op = 0x89, sraiw_op = 0x91, rotriw_op = 0x99, + rcriw_op = 0xa1, +}; + +enum reg2ui6_op { + sllid_op = 0x41, srlid_op = 0x45, sraid_op = 0x49, rotrid_op = 0x4d, + rcrid_op = 0x51, +}; + +enum reg2ui12_op { + andi_op = 0xd, ori_op, xori_op, +}; + +enum reg2lsbw_op { + bstrinsw_op = 0x3, bstrpickw_op = 0x3, +}; + +enum reg2lsbd_op { + bstrinsd_op = 0x2, bstrpickd_op = 0x3, +}; + +enum reg2i8_op { + lddir_op = 0x190, ldpte_op, +}; + +enum reg2i8idx1_op { + vstelmd_op = 0x622, +}; + +enum reg2i8idx2_op { + vstelmw_op = 0x312, xvstelmd_op = 0x331, +}; + +enum reg2i8idx3_op { + vstelmh_op = 0x18a, xvstelmw_op = 0x199, +}; + +enum reg2i8idx4_op { + vstelmb_op = 0xc6, xvstelmh_op = 0xcd, +}; + +enum reg2i8idx5_op { + xvstelmb_op = 0x67, +}; + +enum reg2i9_op { + vldrepld_op = 0x602, xvldrepld_op = 0x642, +}; + +enum reg2i10_op { + vldreplw_op = 0x302, xvldreplw_op = 0x322, +}; + +enum reg2i11_op { + vldreplh_op = 0x182, xvldreplh_op = 0x192, +}; + +enum reg2i12_op { + slti_op = 0x8, sltui_op, addiw_op, addid_op, + lu52id_op, cacop_op = 0x18, xvldreplb_op = 0xca, + ldb_op = 0xa0, ldh_op, ldw_op, ldd_op, stb_op, sth_op, + stw_op, std_op, ldbu_op, ldhu_op, ldwu_op, preld_op, + flds_op, fsts_op, fldd_op, fstd_op, vld_op, vst_op, xvld_op, + xvst_op, ldlw_op = 0xb8, ldrw_op, ldld_op, ldrd_op, stlw_op, + strw_op, stld_op, strd_op, vldreplb_op = 0xc2, +}; + +enum reg2i14_op { + llw_op = 0x20, scw_op, lld_op, scd_op, ldptrw_op, stptrw_op, + ldptrd_op, stptrd_op, +}; + +enum reg2i16_op { + addu16id_op = 0x4, jirl_op = 0x13, beq_op = 0x16, bne_op, blt_op, bge_op, bltu_op, bgeu_op, +}; + +enum reg2csr_op { + csrrd_op = 0x4, csrwr_op = 0x4, csrxchg_op = 0x4, + gcsrrd_op = 0x5, gcsrwr_op = 0x5, gcsrxchg_op = 0x5, +}; + +enum reg3_op { + asrtled_op = 0x2, asrtgtd_op, + addw_op = 0x20, addd_op, subw_op, subd_op, + slt_op, sltu_op, maskeqz_op, masknez_op, + nor_op, and_op, or_op, xor_op, orn_op, + andn_op, sllw_op, srlw_op, sraw_op, slld_op, + srld_op, srad_op, rotrb_op, rotrh_op, + rotrw_op, rotrd_op, mulw_op, mulhw_op, + mulhwu_op, muld_op, mulhd_op, mulhdu_op, + mulwdw_op, mulwdwu_op, divw_op, modw_op, + divwu_op, modwu_op, divd_op, modd_op, + divdu_op, moddu_op, crcwbw_op, + crcwhw_op, crcwww_op, crcwdw_op, crccwbw_op, + crccwhw_op, crccwww_op, crccwdw_op, addu12iw_op, + addu12id_op, + adcb_op = 0x60, adch_op, adcw_op, adcd_op, + sbcb_op, sbch_op, sbcw_op, sbcd_op, + rcrb_op, rcrh_op, rcrw_op, rcrd_op, + ldxb_op = 0x7000, ldxh_op = 0x7008, ldxw_op = 0x7010, ldxd_op = 0x7018, + stxb_op = 0x7020, stxh_op = 0x7028, stxw_op = 0x7030, stxd_op = 0x7038, + ldxbu_op = 0x7040, ldxhu_op = 0x7048, ldxwu_op = 0x7050, + preldx_op = 0x7058, fldxs_op = 0x7060, fldxd_op = 0x7068, + fstxs_op = 0x7070, fstxd_op = 0x7078, vldx_op = 0x7080, + vstx_op = 0x7088, xvldx_op = 0x7090, xvstx_op = 0x7098, + amswapw_op = 0x70c0, amswapd_op, amaddw_op, amaddd_op, amandw_op, + amandd_op, amorw_op, amord_op, amxorw_op, amxord_op, ammaxw_op, + ammaxd_op, amminw_op, ammind_op, ammaxwu_op, ammaxdu_op, + amminwu_op, ammindu_op, amswap_dbw_op, amswap_dbd_op, amadd_dbw_op, + amadd_dbd_op, amand_dbw_op, amand_dbd_op, amor_dbw_op, amor_dbd_op, + amxor_dbw_op, amxor_dbd_op, ammax_dbw_op, ammax_dbd_op, ammin_dbw_op, + ammin_dbd_op, ammax_dbwu_op, ammax_dbdu_op, ammin_dbwu_op, + ammin_dbdu_op, fldgts_op = 0x70e8, fldgtd_op, + fldles_op, fldled_op, fstgts_op, fstgtd_op, fstles_op, fstled_op, + ldgtb_op, ldgth_op, ldgtw_op, ldgtd_op, ldleb_op, ldleh_op, ldlew_op, + ldled_op, stgtb_op, stgth_op, stgtw_op, stgtd_op, stleb_op, stleh_op, + stlew_op, stled_op, +}; + +enum reg3sa2_op { + alslw_op = 0x2, alslwu_op, bytepickw_op, alsld_op = 0x16, + +}; + +enum reg3sa3_op { + bytepickd_op = 0x3, +}; + +struct reg2_format { + __BITFIELD_FIELD(unsigned int opcode : 22, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))) +}; + +struct reg2ui3_format { + __BITFIELD_FIELD(unsigned int opcode : 19, + __BITFIELD_FIELD(unsigned int simmediate : 3, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2ui4_format { + __BITFIELD_FIELD(unsigned int opcode : 18, + __BITFIELD_FIELD(unsigned int simmediate : 4, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2ui5_format { + __BITFIELD_FIELD(unsigned int opcode : 17, + __BITFIELD_FIELD(unsigned int simmediate : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2ui6_format { + __BITFIELD_FIELD(unsigned int opcode : 16, + __BITFIELD_FIELD(unsigned int simmediate : 6, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2lsbw_format { + __BITFIELD_FIELD(unsigned int opcode : 11, + __BITFIELD_FIELD(unsigned int msbw : 5, + __BITFIELD_FIELD(unsigned int op : 1, + __BITFIELD_FIELD(unsigned int lsbw : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))))) +}; + +struct reg2lsbd_format { + __BITFIELD_FIELD(unsigned int opcode : 10, + __BITFIELD_FIELD(unsigned int msbd : 6, + __BITFIELD_FIELD(unsigned int lsbd : 6, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg3_format { + __BITFIELD_FIELD(unsigned int opcode : 17, + __BITFIELD_FIELD(unsigned int rk : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg3sa2_format { + __BITFIELD_FIELD(unsigned int opcode : 15, + __BITFIELD_FIELD(unsigned int simmediate : 2, + __BITFIELD_FIELD(unsigned int rk : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg3sa3_format { + __BITFIELD_FIELD(unsigned int opcode : 14, + __BITFIELD_FIELD(unsigned int simmediate : 3, + __BITFIELD_FIELD(unsigned int rk : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg3sa4_format { + __BITFIELD_FIELD(unsigned int opcode : 13, + __BITFIELD_FIELD(unsigned int simmediate : 4, + __BITFIELD_FIELD(unsigned int rk : 5, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg4_format { + __BITFIELD_FIELD(unsigned int opcode : 12, + __BITFIELD_FIELD(unsigned int fa : 5, + __BITFIELD_FIELD(unsigned int fk : 5, + __BITFIELD_FIELD(unsigned int fj : 5, + __BITFIELD_FIELD(unsigned int fd : 5, + ;))))) +}; + +struct reg2i8_format { + __BITFIELD_FIELD(unsigned int opcode : 14, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i8idx1_format { + __BITFIELD_FIELD(unsigned int opcode : 13, + __BITFIELD_FIELD(unsigned int idx : 1, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg2i8idx2_format { + __BITFIELD_FIELD(unsigned int opcode : 12, + __BITFIELD_FIELD(unsigned int idx : 2, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg2i8idx3_format { + __BITFIELD_FIELD(unsigned int opcode : 11, + __BITFIELD_FIELD(unsigned int idx : 3, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg2i8idx4_format { + __BITFIELD_FIELD(unsigned int opcode : 10, + __BITFIELD_FIELD(unsigned int idx : 4, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg2i8idx5_format { + __BITFIELD_FIELD(unsigned int opcode : 9, + __BITFIELD_FIELD(unsigned int idx : 5, + __BITFIELD_FIELD(unsigned int simmediate : 8, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))))) +}; + +struct reg2i9_format { + __BITFIELD_FIELD(unsigned int opcode : 13, + __BITFIELD_FIELD(unsigned int simmediate : 9, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i10_format { + __BITFIELD_FIELD(unsigned int opcode : 12, + __BITFIELD_FIELD(unsigned int simmediate : 10, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i11_format { + __BITFIELD_FIELD(unsigned int opcode : 11, + __BITFIELD_FIELD(unsigned int simmediate : 11, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i12_format { + __BITFIELD_FIELD(unsigned int opcode : 10, + __BITFIELD_FIELD(signed int simmediate : 12, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2ui12_format { + __BITFIELD_FIELD(unsigned int opcode : 10, + __BITFIELD_FIELD(unsigned int simmediate : 12, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i14_format { + __BITFIELD_FIELD(unsigned int opcode : 8, + __BITFIELD_FIELD(unsigned int simmediate : 14, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2i16_format { + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int simmediate : 16, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg2csr_format { + __BITFIELD_FIELD(unsigned int opcode : 8, + __BITFIELD_FIELD(unsigned int csr : 14, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + ;)))) +}; + +struct reg1i21_format { + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int simmediate_l : 16, + __BITFIELD_FIELD(unsigned int rj : 5, + __BITFIELD_FIELD(unsigned int simmediate_h : 5, + ;)))) +}; + +struct reg1i20_format { + __BITFIELD_FIELD(unsigned int opcode : 7, + __BITFIELD_FIELD(unsigned int simmediate : 20, + __BITFIELD_FIELD(unsigned int rd : 5, + ;))) +}; + +struct reg0i15_format { + __BITFIELD_FIELD(unsigned int opcode : 17, + __BITFIELD_FIELD(unsigned int simmediate : 15, + ;)) +}; + +struct reg0i26_format { + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int simmediate_l : 16, + __BITFIELD_FIELD(unsigned int simmediate_h : 10, + ;))) +}; + +union loongarch_instruction { + unsigned int word; + unsigned short halfword[2]; + unsigned char byte[4]; + struct reg2_format reg2_format; + struct reg2ui3_format reg2ui3_format; + struct reg2ui4_format reg2ui4_format; + struct reg2ui5_format reg2ui5_format; + struct reg2ui6_format reg2ui6_format; + struct reg2ui12_format reg2ui12_format; + struct reg2lsbw_format reg2lsbw_format; + struct reg2lsbd_format reg2lsbd_format; + struct reg3_format reg3_format; + struct reg3sa2_format reg3sa2_format; + struct reg3sa3_format reg3sa3_format; + struct reg3sa4_format reg3sa4_format; + struct reg4_format reg4_format; + struct reg2i8_format reg2i8_format; + struct reg2i8idx1_format reg2i8idx1_format; + struct reg2i8idx2_format reg2i8idx2_format; + struct reg2i8idx3_format reg2i8idx3_format; + struct reg2i8idx4_format reg2i8idx4_format; + struct reg2i8idx5_format reg2i8idx5_format; + struct reg2i9_format reg2i9_format; + struct reg2i10_format reg2i10_format; + struct reg2i11_format reg2i11_format; + struct reg2i12_format reg2i12_format; + struct reg2i14_format reg2i14_format; + struct reg2i16_format reg2i16_format; + struct reg2csr_format reg2csr_format; + struct reg1i21_format reg1i21_format; + struct reg1i20_format reg1i20_format; + struct reg0i15_format reg0i15_format; + struct reg0i26_format reg0i26_format; +}; + +#endif /* _UAPI_ASM_INST_H */ diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h new file mode 100644 index 0000000000000000000000000000000000000000..7f2d99407e453a431e54c8a854bf4a5befcaa44a --- /dev/null +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technologies, Inc. All rights reserved. + * Authors: Sanjay Lal + * Authors: Xing Li + */ + +#ifndef __LINUX_KVM_LOONGARCH_H +#define __LINUX_KVM_LOONGARCH_H + +#include + +#ifndef __KERNEL__ +#include +#endif + +#define __KVM_HAVE_GUEST_DEBUG +#define KVM_GUESTDBG_USE_SW_BP 0x00010000 +#define KVM_GUESTDBG_USE_HW_BP 0x00020000 +#define KVM_DATA_HW_BREAKPOINT_NUM 8 +#define KVM_INST_HW_BREAKPOINT_NUM 8 + +/* + * KVM Loongarch specific structures and definitions. + * + * Some parts derived from the x86 version of this file. + */ + +#define __KVM_HAVE_READONLY_MEM + +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 + +/* + * for KVM_GET_REGS and KVM_SET_REGS + */ +struct kvm_regs { + /* out (KVM_GET_REGS) / in (KVM_SET_REGS) */ + __u64 gpr[32]; + __u64 pc; +}; + +/* + * for KVM_GET_CPUCFG + */ +struct kvm_cpucfg { + /* out (KVM_GET_CPUCFG) */ + __u32 cpucfg[64]; +}; + +/* + * for KVM_GET_FPU and KVM_SET_FPU + */ +struct kvm_fpu { + __u32 fcsr; + __u64 fcc; /* 8x8 */ + struct kvm_fpureg { + __u64 val64[4]; //support max 256 bits + } fpr[32]; +}; + +/* + * For LOONGARCH, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various + * registers. The id field is broken down as follows: + * + * bits[63..52] - As per linux/kvm.h + * bits[51..32] - Must be zero. + * bits[31..16] - Register set. + * + * Register set = 0: GP registers from kvm_regs (see definitions below). + * Register set = 1: CSR registers. + * Register set = 2: KVM specific registers (see definitions below). + * Register set = 3: FPU / SIMD registers (see definitions below). + * Register set = 4: LBT registers (see definitions below). + * + * Other sets registers may be added in the future. Each set would + * have its own identifier in bits[31..16]. + */ + +#define KVM_REG_LOONGARCH_GP (KVM_REG_LOONGARCH | 0x00000ULL) +#define KVM_REG_LOONGARCH_CSR (KVM_REG_LOONGARCH | 0x10000ULL) +#define KVM_REG_LOONGARCH_KVM (KVM_REG_LOONGARCH | 0x20000ULL) +#define KVM_REG_LOONGARCH_FPU (KVM_REG_LOONGARCH | 0x30000ULL) +#define KVM_REG_LOONGARCH_LBT (KVM_REG_LOONGARCH | 0x40000ULL) +#define KVM_REG_LOONGARCH_MASK (KVM_REG_LOONGARCH | 0x70000ULL) +#define KVM_CSR_IDX_MASK (0x10000 - 1) + +/* + * KVM_REG_LOONGARCH_KVM - KVM specific control registers. + */ +#define KVM_REG_LOONGARCH_COUNTER (KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 3) +#define KVM_REG_LOONGARCH_VCPU_RESET (KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 4) + +#define KVM_REG_LBT_SCR0 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 1) +#define KVM_REG_LBT_SCR1 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 2) +#define KVM_REG_LBT_SCR2 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 3) +#define KVM_REG_LBT_SCR3 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 4) +#define KVM_REG_LBT_FLAGS (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 5) +#define KVM_REG_LBT_FTOP (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 6) + +#define __KVM_HAVE_IRQ_LINE + +struct kvm_debug_exit_arch { + __u64 era; + __u32 fwps; + __u32 mwps; + __u32 exception; +}; + +/* for KVM_SET_GUEST_DEBUG */ +struct hw_breakpoint { + __u64 addr; + __u64 mask; + __u32 asid; + __u32 ctrl; +}; + +struct kvm_guest_debug_arch { + struct hw_breakpoint data_breakpoint[KVM_DATA_HW_BREAKPOINT_NUM]; + struct hw_breakpoint inst_breakpoint[KVM_INST_HW_BREAKPOINT_NUM]; + int inst_bp_nums, data_bp_nums; +}; + +/* definition of registers in kvm_run */ +struct kvm_sync_regs { +}; + +/* dummy definition */ +struct kvm_sregs { +}; + +struct kvm_iocsr_entry { + __u32 addr; + __u32 pad; + __u64 data; +}; + +struct kvm_csr_entry { + __u32 index; + __u32 reserved; + __u64 data; +}; + +/* for KVM_GET_MSRS and KVM_SET_MSRS */ +struct kvm_msrs { + __u32 ncsrs; /* number of msrs in entries */ + __u32 pad; + + struct kvm_csr_entry entries[0]; +}; + +struct kvm_loongarch_interrupt { + /* in */ + __u32 cpu; + __u32 irq; +}; + +#define KVM_IRQCHIP_LS7A_IOAPIC 0x0 +#define KVM_IRQCHIP_LS3A_GIPI 0x1 +#define KVM_IRQCHIP_LS3A_HT_IRQ 0x2 +#define KVM_IRQCHIP_LS3A_ROUTE 0x3 +#define KVM_IRQCHIP_LS3A_EXTIRQ 0x4 +#define KVM_IRQCHIP_LS3A_IPMASK 0x5 +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 64 + +#define KVM_MAX_CORES 256 +#define KVM_EXTIOI_IRQS (256) +#define KVM_EXTIOI_IRQS_BITMAP_SIZE (KVM_EXTIOI_IRQS / 8) +/* map to ipnum per 32 irqs */ +#define KVM_EXTIOI_IRQS_IPMAP_SIZE (KVM_EXTIOI_IRQS / 32) +#define KVM_EXTIOI_IRQS_PER_GROUP 32 +#define KVM_EXTIOI_IRQS_COREMAP_SIZE (KVM_EXTIOI_IRQS) +#define KVM_EXTIOI_IRQS_NODETYPE_SIZE 16 + +struct ls7a_ioapic_state { + /* 0x000 interrupt id register */ + __u64 int_id; + /* 0x020 interrupt mask register */ + __u64 int_mask; + /* 0x040 1=msi */ + __u64 htmsi_en; + /* 0x060 edge=1 level =0 */ + __u64 intedge; + /* 0x080 for clean edge int,set 1 clean,set 0 is noused */ + __u64 intclr; + /* 0x0c0 */ + __u64 auto_crtl0; + /* 0x0e0 */ + __u64 auto_crtl1; + /* 0x100 - 0x140 */ + __u8 route_entry[64]; + /* 0x200 - 0x240 */ + __u8 htmsi_vector[64]; + /* 0x300 */ + __u64 intisr_chip0; + /* 0x320 */ + __u64 intisr_chip1; + /* edge detection */ + __u64 last_intirr; + /* 0x380 interrupt request register */ + __u64 intirr; + /* 0x3a0 interrupt service register */ + __u64 intisr; + /* 0x3e0 interrupt level polarity selection register, + * 0 for high level tirgger + */ + __u64 int_polarity; +}; + +struct loongarch_gipi_single { + __u32 status; + __u32 en; + __u32 set; + __u32 clear; + __u64 buf[4]; +}; + +struct loongarch_gipiState { + struct loongarch_gipi_single core[KVM_MAX_CORES]; +}; + +struct kvm_loongarch_ls3a_extirq_state { + union ext_en_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_en_r; + union bounce_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } bounce_r; + union ext_isr_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_isr_r; + union ext_core_isr_r { + uint64_t reg_u64[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_core_isr_r; + union ip_map_r { + uint64_t reg_u64; + uint32_t reg_u32[KVM_EXTIOI_IRQS_IPMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_IPMAP_SIZE]; + } ip_map_r; + union core_map_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_COREMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_COREMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_COREMAP_SIZE]; + } core_map_r; + union node_type_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 4]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 2]; + uint16_t reg_u16[KVM_EXTIOI_IRQS_NODETYPE_SIZE]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_NODETYPE_SIZE * 2]; + } node_type_r; +}; + +struct loongarch_kvm_irqchip { + __u16 chip_id; + __u16 len; + __u16 vcpu_id; + __u16 reserved; + char data[0]; +}; + +#endif /* __LINUX_KVM_LOONGARCH_H */ diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..4013470c389ea3c6eb4e593a0efc8a4251a6b393 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/kvm_para.h @@ -0,0 +1,8 @@ +#ifndef _UAPI_ASM_LOONGARCH_KVM_PARA_H +#define _UAPI_ASM_LOONGARCH_KVM_PARA_H + +/* Device Control API on vcpu fd */ +#define KVM_LARCH_VCPU_PVTIME_CTRL 2 +#define KVM_LARCH_VCPU_PVTIME_IPA 0 + +#endif /* _UAPI_ASM_LOONGARCH_KVM_PARA_H */ diff --git a/arch/loongarch/include/uapi/asm/perf_regs.h b/arch/loongarch/include/uapi/asm/perf_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..29d69c00fc7a665f743d931223916c43cd0b4849 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/perf_regs.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASM_LOONGARCH_PERF_REGS_H +#define _ASM_LOONGARCH_PERF_REGS_H + +enum perf_event_loongarch_regs { + PERF_REG_LOONGARCH_PC, + PERF_REG_LOONGARCH_R1, + PERF_REG_LOONGARCH_R2, + PERF_REG_LOONGARCH_R3, + PERF_REG_LOONGARCH_R4, + PERF_REG_LOONGARCH_R5, + PERF_REG_LOONGARCH_R6, + PERF_REG_LOONGARCH_R7, + PERF_REG_LOONGARCH_R8, + PERF_REG_LOONGARCH_R9, + PERF_REG_LOONGARCH_R10, + PERF_REG_LOONGARCH_R11, + PERF_REG_LOONGARCH_R12, + PERF_REG_LOONGARCH_R13, + PERF_REG_LOONGARCH_R14, + PERF_REG_LOONGARCH_R15, + PERF_REG_LOONGARCH_R16, + PERF_REG_LOONGARCH_R17, + PERF_REG_LOONGARCH_R18, + PERF_REG_LOONGARCH_R19, + PERF_REG_LOONGARCH_R20, + PERF_REG_LOONGARCH_R21, + PERF_REG_LOONGARCH_R22, + PERF_REG_LOONGARCH_R23, + PERF_REG_LOONGARCH_R24, + PERF_REG_LOONGARCH_R25, + PERF_REG_LOONGARCH_R26, + PERF_REG_LOONGARCH_R27, + PERF_REG_LOONGARCH_R28, + PERF_REG_LOONGARCH_R29, + PERF_REG_LOONGARCH_R30, + PERF_REG_LOONGARCH_R31, + PERF_REG_LOONGARCH_MAX, +}; +#endif /* _ASM_LOONGARCH_PERF_REGS_H */ diff --git a/arch/loongarch/include/uapi/asm/ptrace.h b/arch/loongarch/include/uapi/asm/ptrace.h new file mode 100644 index 0000000000000000000000000000000000000000..cbde1deb15ebcb8d78270b9c26162f4c3962df75 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/ptrace.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +* Huacai Chen +*/ +#ifndef _UAPI_ASM_PTRACE_H +#define _UAPI_ASM_PTRACE_H + +#include + +#ifndef __KERNEL__ +#include +#endif + +/* + * For PTRACE_{POKE,PEEK}USR. 0 - 31 are GPRs, + * 32 is syscall's original ARG0, 33 is PC, 34 is BADVADDR. + */ +#define GPR_BASE 0 +#define GPR_NUM 32 +#define GPR_END (GPR_BASE + GPR_NUM - 1) +#define ARG0 (GPR_END + 1) +#define PC (GPR_END + 2) +#define BADVADDR (GPR_END + 3) + +#define NUM_FPU_REGS 32 + +struct user_pt_regs { + /* Main processor registers. */ + unsigned long regs[32]; + + /* Original syscall arg0. */ + unsigned long orig_a0; + + /* Special CSR registers. */ + unsigned long csr_era; + unsigned long csr_badv; + unsigned long reserved[10]; +} __attribute__((aligned(8))); + +struct user_fp_state { + uint64_t fpr[32]; + uint64_t fcc; + uint32_t fcsr; +}; + +struct user_lsx_state { + /* 32 registers, 128 bits width per register. */ + uint64_t vregs[32*2]; +}; + +struct user_lasx_state { + /* 32 registers, 256 bits width per register. */ + uint64_t vregs[32*4]; +}; + +/* + * This structure definition saves the LBT data structure, + * the data comes from the task_struct structure, format is as follows: + * regs[0]: thread.lbt.scr0 + * regs[1]: thread.lbt.scr1 + * regs[2]: thread.lbt.scr2 + * regs[3]: thread.lbt.scr3 + * regs[4]: thread.lbt.eflags + * regs[5]: thread.fpu.ftop + */ +struct user_lbt_state { + uint64_t regs[6]; +}; + +/* Read and write watchpoint registers. */ +#define NUM_WATCH_REGS 16 + +enum pt_watch_style { + pt_watch_style_la32, + pt_watch_style_la64 +}; + +struct la32_watch_regs { + uint32_t addr; + uint32_t mask; + /* irw/irwsta/irwmask I R W bits. + * bit 0 -- 1 if W bit is usable. + * bit 1 -- 1 if R bit is usable. + * bit 2 -- 1 if I bit is usable. + */ + uint8_t irw; + uint8_t irwstat; + uint8_t irwmask; +} __attribute__((aligned(8))); + +struct la64_watch_regs { + uint64_t addr; + uint64_t mask; + /* irw/irwsta/irwmask I R W bits. + * bit 0 -- 1 if W bit is usable. + * bit 1 -- 1 if R bit is usable. + * bit 2 -- 1 if I bit is usable. + */ + uint8_t irw; + uint8_t irwstat; + uint8_t irwmask; +} __attribute__((aligned(8))); + +struct pt_watch_regs { + int16_t max_valid; + int16_t num_valid; + enum pt_watch_style style; + union { + struct la32_watch_regs la32[NUM_WATCH_REGS]; + struct la64_watch_regs la64[NUM_WATCH_REGS]; + }; +}; + +#define PTRACE_SYSEMU 0x1f +#define PTRACE_SYSEMU_SINGLESTEP 0x20 +#define PTRACE_GET_WATCH_REGS 0xd0 +#define PTRACE_SET_WATCH_REGS 0xd1 + +/* Watch irw/irwmask/irwstat bit definitions */ +#define LA_WATCH_W (1 << 0) +#define LA_WATCH_R (1 << 1) +#define LA_WATCH_I (1 << 2) +#define LA_WATCH_IRW (LA_WATCH_W | LA_WATCH_R | LA_WATCH_I) + +#endif /* _UAPI_ASM_PTRACE_H */ diff --git a/arch/loongarch/include/uapi/asm/reg.h b/arch/loongarch/include/uapi/asm/reg.h new file mode 100644 index 0000000000000000000000000000000000000000..24b79dbd3bee73562544758b349e5d15a59aa3db --- /dev/null +++ b/arch/loongarch/include/uapi/asm/reg.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Various register offset definitions for debuggers, core file + * examiners and whatnot. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#ifndef __UAPI_ASM_LOONGARCH_REG_H +#define __UAPI_ASM_LOONGARCH_REG_H + +#define LOONGARCH_EF_R0 0 +#define LOONGARCH_EF_R1 1 +#define LOONGARCH_EF_R2 2 +#define LOONGARCH_EF_R3 3 +#define LOONGARCH_EF_R4 4 +#define LOONGARCH_EF_R5 5 +#define LOONGARCH_EF_R6 6 +#define LOONGARCH_EF_R7 7 +#define LOONGARCH_EF_R8 8 +#define LOONGARCH_EF_R9 9 +#define LOONGARCH_EF_R10 10 +#define LOONGARCH_EF_R11 11 +#define LOONGARCH_EF_R12 12 +#define LOONGARCH_EF_R13 13 +#define LOONGARCH_EF_R14 14 +#define LOONGARCH_EF_R15 15 +#define LOONGARCH_EF_R16 16 +#define LOONGARCH_EF_R17 17 +#define LOONGARCH_EF_R18 18 +#define LOONGARCH_EF_R19 19 +#define LOONGARCH_EF_R20 20 +#define LOONGARCH_EF_R21 21 +#define LOONGARCH_EF_R22 22 +#define LOONGARCH_EF_R23 23 +#define LOONGARCH_EF_R24 24 +#define LOONGARCH_EF_R25 25 +#define LOONGARCH_EF_R26 26 +#define LOONGARCH_EF_R27 27 +#define LOONGARCH_EF_R28 28 +#define LOONGARCH_EF_R29 29 +#define LOONGARCH_EF_R30 30 +#define LOONGARCH_EF_R31 31 + +/* + * Saved special registers + */ +#define LOONGARCH_EF_ORIG_A0 32 +#define LOONGARCH_EF_CSR_ERA 33 +#define LOONGARCH_EF_CSR_BADV 34 +#define LOONGARCH_EF_CSR_CRMD 35 +#define LOONGARCH_EF_CSR_PRMD 36 +#define LOONGARCH_EF_CSR_EUEN 37 +#define LOONGARCH_EF_CSR_ECFG 38 +#define LOONGARCH_EF_CSR_ESTAT 39 + +#define LOONGARCH_EF_SIZE 320 /* size in bytes */ + +#endif /* __UAPI_ASM_LOONGARCH_REG_H */ diff --git a/arch/loongarch/include/uapi/asm/sigcontext.h b/arch/loongarch/include/uapi/asm/sigcontext.h new file mode 100644 index 0000000000000000000000000000000000000000..67a459776784817575957eb29b9e2cf8d204f147 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/sigcontext.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ +#ifndef _UAPI_ASM_SIGCONTEXT_H +#define _UAPI_ASM_SIGCONTEXT_H + +#include +#include + +/* FP context was used */ +#define SC_USED_FP (1 << 0) +/* Address error was due to memory load */ +#define SC_ADDRERR_RD (1 << 30) +/* Address error was due to memory store */ +#define SC_ADDRERR_WR (1 << 31) + +struct sigcontext { + __u64 sc_pc; + __u64 sc_regs[32]; + __u32 sc_flags; + __u64 sc_extcontext[0] __attribute__((__aligned__(16))); +}; + +#define CONTEXT_INFO_ALIGN 16 +struct sctx_info { + __u32 magic; + __u32 size; + __u64 padding; /* padding to 16 bytes */ +}; + +/* FPU context */ +#define FPU_CTX_MAGIC 0x46505501 +#define FPU_CTX_ALIGN 8 +struct fpu_context { + __u64 regs[32]; + __u64 fcc; + __u32 fcsr; +}; + +/* LSX context */ +#define LSX_CTX_MAGIC 0x53580001 +#define LSX_CTX_ALIGN 16 +struct lsx_context { + __u64 regs[2*32]; + __u64 fcc; + __u32 fcsr; +}; + +/* LASX context */ +#define LASX_CTX_MAGIC 0x41535801 +#define LASX_CTX_ALIGN 32 +struct lasx_context { + __u64 regs[4*32]; + __u64 fcc; + __u32 fcsr; +}; + +/* LBT context */ +#define LBT_CTX_MAGIC 0x42540001 +#define LBT_CTX_ALIGN 8 +struct lbt_context { + __u64 scr[4]; + __u32 eflags; +}; + +#endif /* _UAPI_ASM_SIGCONTEXT_H */ diff --git a/arch/loongarch/include/uapi/asm/signal.h b/arch/loongarch/include/uapi/asm/signal.h new file mode 100644 index 0000000000000000000000000000000000000000..c1333dc49f8244c7a3e72ae22f98c12a4e127011 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/signal.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ +#ifndef _UAPI_ASM_SIGNAL_H +#define _UAPI_ASM_SIGNAL_H + +#define MINSIGSTKSZ 4096 +#define SIGSTKSZ 16384 + +#include + +#endif diff --git a/arch/loongarch/include/uapi/asm/swab.h b/arch/loongarch/include/uapi/asm/swab.h new file mode 100644 index 0000000000000000000000000000000000000000..ea16e98e18a6292de699312bc7fa38e9cb153b56 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/swab.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technologies, Inc. All rights reserved. + * Authors: Jun Yi + */ +#ifndef _ASM_SWAB_H +#define _ASM_SWAB_H + +#include +#include + +#define __SWAB_64_THRU_32__ + +static inline __attribute_const__ __u16 __arch_swab16(__u16 x) +{ + __asm__( + " revb.2h %0, %1 \n" + : "=r" (x) + : "r" (x)); + + return x; +} +#define __arch_swab16 __arch_swab16 + +static inline __attribute_const__ __u32 __arch_swab32(__u32 x) +{ + __asm__( + " revb.2h %0, %1 \n" + " rotri.w %0, %0, 16 \n" + : "=r" (x) + : "r" (x)); + + return x; +} +#define __arch_swab32 __arch_swab32 + +#ifdef __loongarch64 +static inline __attribute_const__ __u64 __arch_swab64(__u64 x) +{ + __asm__( + " revb.4h %0, %1 \n" + " revh.d %0, %0 \n" + : "=r" (x) + : "r" (x)); + + return x; +} +#define __arch_swab64 __arch_swab64 +#endif /* __loongarch64 */ +#endif /* _ASM_SWAB_H */ diff --git a/arch/loongarch/include/uapi/asm/ucontext.h b/arch/loongarch/include/uapi/asm/ucontext.h new file mode 100644 index 0000000000000000000000000000000000000000..12577e22b1c710e43d9d6d25e29eef55792f2b99 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/ucontext.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __LOONGARCH_UAPI_ASM_UCONTEXT_H +#define __LOONGARCH_UAPI_ASM_UCONTEXT_H + +/** + * struct ucontext - user context structure + * @uc_flags: + * @uc_link: + * @uc_stack: + * @uc_mcontext: holds basic processor state + * @uc_sigmask: + * @uc_extcontext: holds extended processor state + */ +struct ucontext { + unsigned long uc_flags; + struct ucontext *uc_link; + stack_t uc_stack; + sigset_t uc_sigmask; + /* There's some padding here to allow sigset_t to be expanded in the + * future. Though this is unlikely, other architectures put uc_sigmask + * at the end of this structure and explicitly state it can be + * expanded, so we didn't want to box ourselves in here. */ + __u8 __unused[1024 / 8 - sizeof(sigset_t)]; + /* We can't put uc_sigmask at the end of this structure because we need + * to be able to expand sigcontext in the future. For example, the + * vector ISA extension will almost certainly add ISA state. We want + * to ensure all user-visible ISA state can be saved and restored via a + * ucontext, so we're putting this at the end in order to allow for + * infinite extensibility. Since we know this will be extended and we + * assume sigset_t won't be extended an extreme amount, we're + * prioritizing this. */ + struct sigcontext uc_mcontext; +}; + +#endif /* __LOONGARCH_UAPI_ASM_UCONTEXT_H */ diff --git a/arch/loongarch/include/uapi/asm/unistd.h b/arch/loongarch/include/uapi/asm/unistd.h new file mode 100644 index 0000000000000000000000000000000000000000..fcb668984f0336fa77740deb7b40b410450cd885 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/unistd.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#define __ARCH_WANT_SYS_CLONE +#define __ARCH_WANT_SYS_CLONE3 + +#include diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9df4750dd5ee3874e7bb95123b1871c905d522c8 --- /dev/null +++ b/arch/loongarch/kernel/Makefile @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux/LoongArch kernel. +# + +OBJECT_FILES_NON_STANDARD_head.o :=y +OBJECT_FILES_NON_STANDARD_relocate_kernel.o :=y + +extra-y := head.o vmlinux.lds + +obj-y += cpu-probe.o elf.o entry.o genex.o idle.o irq.o \ + process.o ptrace.o reset.o setup.o signal.o io.o \ + syscall.o time.o topology.o traps.o switch.o \ + cacheinfo.o cmpxchg.o vdso.o alternative.o inst.o + +obj-$(CONFIG_ACPI) += acpi.o +obj-$(CONFIG_EFI) += efi.o + +obj-$(CONFIG_ARCH_STRICT_ALIGN) += unaligned.o + +CFLAGS_module.o += $(call cc-option,-Wno-override-init,) +CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,) +CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,) + +ifdef CONFIG_FUNCTION_TRACER + ifndef CONFIG_DYNAMIC_FTRACE + obj-y += mcount.o ftrace.o + CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) + else + obj-y += mcount_dyn.o ftrace_dyn.o + CFLAGS_REMOVE_ftrace_dyn.o = $(CC_FLAGS_FTRACE) + endif + CFLAGS_REMOVE_inst.o = $(CC_FLAGS_FTRACE) + CFLAGS_REMOVE_time.o = $(CC_FLAGS_FTRACE) + CFLAGS_REMOVE_perf_event.o = $(CC_FLAGS_FTRACE) +endif + +KASAN_SANITIZE_efi.o := n +KASAN_SANITIZE_cpu-probe.o := n +KASAN_SANITIZE_traps.o := n +KASAN_SANITIZE_smp.o := n +KASAN_SANITIZE_vdso.o := n +KASAN_SANITIZE_watch.o := n + +obj-$(CONFIG_MODULES) += module.o module-sections.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o + +obj-$(CONFIG_CPU_HAS_FPU) += fpu.o kfpu.o + +obj-$(CONFIG_SMP) += smp.o + +obj-$(CONFIG_CPU_HAS_LBT) += scall-trans-mips64.o trans_mips_syscalls.o scall-trans-i386.o trans_i386_syscalls.o + +obj-$(CONFIG_PROC_FS) += proc.o +obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o + +obj-$(CONFIG_RELOCATABLE) += relocate.o + +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o + +obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o +obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o +obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o + +obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o +obj-$(CONFIG_HARDWARE_WATCHPOINTS) += watch.o + +obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_UPROBES) += uprobes.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o +obj-$(CONFIG_SPINLOCK_TEST) += spinlock_test.o + +obj-$(CONFIG_PARAVIRT) += paravirt.o +obj-$(CONFIG_PARAVIRT_SPINLOCKS) += paravirt-spinlocks.o + +CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/access-helper.h b/arch/loongarch/kernel/access-helper.h new file mode 100644 index 0000000000000000000000000000000000000000..4a35ca81bd08e6a77ea09c39d269ab5cbd9b6180 --- /dev/null +++ b/arch/loongarch/kernel/access-helper.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +static inline int __get_inst(u32 *i, u32 *p, bool user) +{ + return user ? get_user(*i, (u32 __user *)p) : get_kernel_nofault(*i, p); +} + +static inline int __get_addr(unsigned long *a, unsigned long *p, bool user) +{ + return user ? get_user(*a, (unsigned long __user *)p) : get_kernel_nofault(*a, p); +} diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c new file mode 100644 index 0000000000000000000000000000000000000000..f44435ec1d6e4f6909a72e5f9a669139a61b51bd --- /dev/null +++ b/arch/loongarch/kernel/acpi.c @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * acpi.c - Architecture-Specific Low-Level ACPI Boot Support + * + * Author: Jianmin Lv + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int acpi_disabled; +EXPORT_SYMBOL(acpi_disabled); +int acpi_noirq; +int acpi_pci_disabled; +EXPORT_SYMBOL(acpi_pci_disabled); +int acpi_strict = 1; /* We have no workarounds on LoongArch */ +int num_processors; +int disabled_cpus; +enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_LPIC; + +u64 acpi_saved_sp; + +#define PREFIX "ACPI: " + +struct acpi_madt_core_pic acpi_core_pic[MAX_CORE_PIC]; + +int acpi_gsi_to_irq(u32 gsi, unsigned int *irqp) +{ + if (irqp != NULL) + *irqp = acpi_register_gsi(NULL, gsi, -1, -1); + return (*irqp >= 0) ? 0 : -EINVAL; +} +EXPORT_SYMBOL_GPL(acpi_gsi_to_irq); + +int acpi_isa_irq_to_gsi(unsigned int isa_irq, u32 *gsi) +{ + if (gsi) + *gsi = isa_irq; + return 0; +} + +/* + * success: return IRQ number (>=0) + * failure: return < 0 + */ +int acpi_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity) +{ + int id; + struct irq_fwspec fwspec; + + switch (gsi) { + case GSI_MIN_CPU_IRQ ... GSI_MAX_CPU_IRQ: + fwspec.fwnode = liointc_domain->fwnode; + fwspec.param[0] = gsi - GSI_MIN_CPU_IRQ; + fwspec.param[1] = acpi_dev_get_irq_type(trigger, polarity); + fwspec.param_count = 2; + + return irq_create_fwspec_mapping(&fwspec); + + case GSI_MIN_LPC_IRQ ... GSI_MAX_LPC_IRQ: + if (!pch_lpc_domain) + return -EINVAL; + + fwspec.fwnode = pch_lpc_domain->fwnode; + fwspec.param[0] = gsi - GSI_MIN_LPC_IRQ; + fwspec.param[1] = acpi_dev_get_irq_type(trigger, polarity); + fwspec.param_count = 2; + + return irq_create_fwspec_mapping(&fwspec); + + case GSI_MIN_PCH_IRQ ... GSI_MAX_PCH_IRQ: + id = find_pch_pic(gsi); + if (id < 0) + return -EINVAL; + + fwspec.fwnode = pch_pic_domain[id]->fwnode; + fwspec.param[0] = gsi - acpi_pchpic[id]->gsi_base; + fwspec.param[1] = acpi_dev_get_irq_type(trigger, polarity); + fwspec.param_count = 2; + + return irq_create_fwspec_mapping(&fwspec); + } + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(acpi_register_gsi); + +void acpi_unregister_gsi(u32 gsi) +{ + +} +EXPORT_SYMBOL_GPL(acpi_unregister_gsi); + +void __init __iomem * __acpi_map_table(unsigned long phys, unsigned long size) +{ + + if (!phys || !size) + return NULL; + + return early_memremap(phys, size); +} +void __init __acpi_unmap_table(void __iomem *map, unsigned long size) +{ + if (!map || !size) + return; + + early_memunmap(map, size); +} + +void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) +{ + if (!memblock_is_memory(phys)) + return ioremap(phys, size); + else + return ioremap_cache(phys, size); +} + +static int set_processor_mask(u32 id, u32 flags) +{ + + int cpu, cpuid = id; + + if (num_processors >= nr_cpu_ids) { + pr_warn(PREFIX "nr_cpus/possible_cpus limit of %i reached." + " processor 0x%x ignored.\n", nr_cpu_ids, cpuid); + + return -ENODEV; + + } + if (cpuid == loongson_sysconf.boot_cpu_id) + cpu = 0; + else + cpu = cpumask_next_zero(-1, cpu_present_mask); + + if (flags & ACPI_MADT_ENABLED) { + num_processors++; + set_cpu_possible(cpu, true); + set_cpu_present(cpu, true); + __cpu_number_map[cpuid] = cpu; + __cpu_logical_map[cpu] = cpuid; + } else + disabled_cpus++; + + return cpu; +} + +static int __init +acpi_parse_cpuintc(union acpi_subtable_headers *header, const unsigned long end) +{ + struct acpi_madt_core_pic *processor = NULL; + + processor = (struct acpi_madt_core_pic *)header; + if (BAD_MADT_ENTRY(processor, end)) + return -EINVAL; + + acpi_table_print_madt_entry(&header->common); + acpi_core_pic[processor->core_id] = *processor; + set_processor_mask(processor->core_id, processor->flags); + + return 0; +} + +static int __init +acpi_parse_liointc(union acpi_subtable_headers *header, const unsigned long end) +{ + struct acpi_madt_lio_pic *liointc = NULL; + + liointc = (struct acpi_madt_lio_pic *)header; + + if (BAD_MADT_ENTRY(liointc, end)) + return -EINVAL; + + acpi_liointc = liointc; + + return 0; +} + +static int __init +acpi_parse_eiointc(union acpi_subtable_headers *header, const unsigned long end) +{ + static int id = 0; + struct acpi_madt_eio_pic *eiointc = NULL; + + eiointc = (struct acpi_madt_eio_pic *)header; + + if (BAD_MADT_ENTRY(eiointc, end)) + return -EINVAL; + + acpi_eiointc[id++] = eiointc; + loongson_sysconf.nr_io_pics = id; + + return 0; +} + +static int __init +acpi_parse_htintc(union acpi_subtable_headers *header, const unsigned long end) +{ + struct acpi_madt_ht_pic *htintc = NULL; + + htintc = (struct acpi_madt_ht_pic *)header; + + if (BAD_MADT_ENTRY(htintc, end)) + return -EINVAL; + + acpi_htintc = htintc; + loongson_sysconf.nr_io_pics = 1; + + return 0; +} + +static int __init +acpi_parse_pch_pic(union acpi_subtable_headers *header, const unsigned long end) +{ + static int id = 0; + struct acpi_madt_bio_pic *pchpic = NULL; + + pchpic = (struct acpi_madt_bio_pic *)header; + + if (BAD_MADT_ENTRY(pchpic, end)) + return -EINVAL; + + acpi_pchpic[id++] = pchpic; + + return 0; +} + +static int __init +acpi_parse_pch_msi(union acpi_subtable_headers *header, const unsigned long end) +{ + static int id = 0; + struct acpi_madt_msi_pic *pchmsi = NULL; + + pchmsi = (struct acpi_madt_msi_pic *)header; + + if (BAD_MADT_ENTRY(pchmsi, end)) + return -EINVAL; + + acpi_pchmsi[id++] = pchmsi; + + return 0; +} + +static int __init +acpi_parse_pch_lpc(union acpi_subtable_headers *header, const unsigned long end) +{ + struct acpi_madt_lpc_pic *pchlpc = NULL; + + pchlpc = (struct acpi_madt_lpc_pic *)header; + + if (BAD_MADT_ENTRY(pchlpc, end)) + return -EINVAL; + + acpi_pchlpc = pchlpc; + + return 0; +} + +static void __init acpi_process_madt(void) +{ + int i, error; + + for (i = 0; i < NR_CPUS; i++) { + __cpu_number_map[i] = -1; + __cpu_logical_map[i] = -1; + } + + /* Parse MADT CPUINTC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_CORE_PIC, acpi_parse_cpuintc, MAX_CORE_PIC); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (CPUINTC entries), ACPI disabled\n"); + return; + } + + loongson_sysconf.nr_cpus = num_processors; + + /* Parse MADT LIOINTC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_LIO_PIC, acpi_parse_liointc, 1); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (LIOINTC entries), ACPI disabled\n"); + return; + } + + /* Parse MADT EIOINTC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_EIO_PIC, acpi_parse_eiointc, MAX_IO_PICS); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (EIOINTC entries), ACPI disabled\n"); + return; + } + + /* Parse MADT HTVEC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_HT_PIC, acpi_parse_htintc, 1); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (HTVEC entries), ACPI disabled\n"); + return; + } + + /* Parse MADT PCHPIC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_BIO_PIC, acpi_parse_pch_pic, MAX_IO_PICS); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (PCHPIC entries), ACPI disabled\n"); + return; + } + + /* Parse MADT PCHMSI entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, acpi_parse_pch_msi, MAX_IO_PICS); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (PCHMSI entries), ACPI disabled\n"); + return; + } + + /* Parse MADT PCHLPC entries */ + error = acpi_table_parse_madt(ACPI_MADT_TYPE_LPC_PIC, acpi_parse_pch_lpc, 1); + if (error < 0) { + disable_acpi(); + pr_err(PREFIX "Invalid BIOS MADT (PCHLPC entries), ACPI disabled\n"); + return; + } +} + +int pptt_enabled; + +int __init parse_acpi_topology(void) +{ + int cpu, topology_id; + + for_each_possible_cpu(cpu) { + topology_id = find_acpi_cpu_topology(cpu, 0); + if (topology_id < 0) { + pr_warn("Invalid BIOS PPTT\n"); + return -ENOENT; + } + + if (acpi_pptt_cpu_is_thread(cpu) <= 0) + cpu_data[cpu].core = topology_id; + else { + topology_id = find_acpi_cpu_topology(cpu, 1); + if (topology_id < 0) + return -ENOENT; + + cpu_data[cpu].core = topology_id; + } + } + + pptt_enabled = 1; + + return 0; +} + +#ifndef CONFIG_SUSPEND +int (*acpi_suspend_lowlevel)(void); +#else +int (*acpi_suspend_lowlevel)(void) = loongarch_acpi_suspend; +#endif + +void __init acpi_boot_table_init(void) +{ + /* + * If acpi_disabled, bail out + */ + if (acpi_disabled) + return; + + /* + * Initialize the ACPI boot-time table parser. + */ + if (acpi_table_init()) { + disable_acpi(); + return; + } + + loongson_sysconf.boot_cpu_id = read_csr_cpuid(); + + /* + * Process the Multiple APIC Description Table (MADT), if present + */ + acpi_process_madt(); + + /* Do not enable ACPI SPCR console by default */ + acpi_parse_spcr(earlycon_acpi_spcr_enable, false); +} + +#ifdef CONFIG_ACPI_NUMA + +static __init int setup_node(int pxm) +{ + return acpi_map_pxm_to_node(pxm); +} + +/* + * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for + * I/O localities since SRAT does not list them. I/O localities are + * not supported at this point. + */ +unsigned int numa_distance_cnt; + +static inline unsigned int get_numa_distances_cnt(struct acpi_table_slit *slit) +{ + return slit->locality_count; +} + +void __init numa_set_distance(int from, int to, int distance) +{ + if ((u8)distance != distance || (from == to && distance != LOCAL_DISTANCE)) { + pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", + from, to, distance); + return; + } + + node_distances[from][to] = distance; +} + +/* Callback for Proximity Domain -> CPUID mapping */ +void __init +acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) +{ + int pxm, node; + + if (srat_disabled()) + return; + if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) { + bad_srat(); + return; + } + if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) + return; + pxm = pa->proximity_domain_lo; + if (acpi_srat_revision >= 2) { + pxm |= (pa->proximity_domain_hi[0] << 8); + pxm |= (pa->proximity_domain_hi[1] << 16); + pxm |= (pa->proximity_domain_hi[2] << 24); + } + node = setup_node(pxm); + if (node < 0) { + printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm); + bad_srat(); + return; + } + + if (pa->apic_id >= CONFIG_NR_CPUS) { + printk(KERN_INFO "SRAT: PXM %u -> CPU 0x%02x -> Node %u skipped apicid that is too big\n", + pxm, pa->apic_id, node); + return; + } + + early_numa_add_cpu(pa->apic_id, node); + + set_cpuid_to_node(pa->apic_id, node); + node_set(node, numa_nodes_parsed); + printk(KERN_INFO "SRAT: PXM %u -> CPU 0x%02x -> Node %u\n", + pxm, pa->apic_id, node); +} + +#endif + +void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size) +{ + memblock_reserve(addr, size); +} + +#ifdef CONFIG_ACPI_HOTPLUG_CPU + +#include + +static int __ref acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) +{ +#ifdef CONFIG_ACPI_NUMA + int nid; + + nid = acpi_get_node(handle); + if (nid != NUMA_NO_NODE) { + set_cpuid_to_node(physid, nid); + node_set(nid, numa_nodes_parsed); + set_cpu_numa_node(cpu, nid); + cpumask_set_cpu(cpu, cpumask_of_node(nid)); + } +#endif + return 0; +} + +int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, int *pcpu) +{ + int cpu; + + cpu = set_processor_mask(physid, ACPI_MADT_ENABLED); + if (cpu < 0) { + pr_info(PREFIX "Unable to map lapic to logical cpu number\n"); + return cpu; + } + + acpi_map_cpu2node(handle, cpu, physid); + + *pcpu = cpu; + + return 0; +} +EXPORT_SYMBOL(acpi_map_cpu); + +int acpi_unmap_cpu(int cpu) +{ +#ifdef CONFIG_ACPI_NUMA + set_cpuid_to_node(cpu_logical_map(cpu), NUMA_NO_NODE); +#endif + set_cpu_present(cpu, false); + num_processors--; + + pr_info("cpu%d hot remove!\n", cpu); + + return 0; +} +EXPORT_SYMBOL(acpi_unmap_cpu); + +#endif /* CONFIG_ACPI_HOTPLUG_CPU */ diff --git a/arch/loongarch/kernel/alternative.c b/arch/loongarch/kernel/alternative.c new file mode 100644 index 0000000000000000000000000000000000000000..c6ed3b40372b71407de75e22b29c5e48ff6e0932 --- /dev/null +++ b/arch/loongarch/kernel/alternative.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +int __read_mostly alternatives_patched; + +EXPORT_SYMBOL_GPL(alternatives_patched); + +#define MAX_PATCH_SIZE (((u8)(-1)) / LOONGARCH_INSN_SIZE) + +static int __initdata_or_module debug_alternative; + +static int __init debug_alt(char *str) +{ + debug_alternative = 1; + return 1; +} +__setup("debug-alternative", debug_alt); + +#define DPRINTK(fmt, args...) \ +do { \ + if (debug_alternative) \ + printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args); \ +} while (0) + +#define DUMP_WORDS(buf, count, fmt, args...) \ +do { \ + if (unlikely(debug_alternative)) { \ + int _j; \ + union loongarch_instruction *_buf = buf; \ + \ + if (!(count)) \ + break; \ + \ + printk(KERN_DEBUG fmt, ##args); \ + for (_j = 0; _j < count - 1; _j++) \ + printk(KERN_CONT "<%08x> ", _buf[_j].word); \ + printk(KERN_CONT "<%08x>\n", _buf[_j].word); \ + } \ +} while (0) + +/* Use this to add nops to a buffer, then text_poke the whole buffer. */ +static void __init_or_module add_nops(union loongarch_instruction *insn, int count) +{ + while (count--) { + insn->word = INSN_NOP; + insn++; + } +} + +/* Is the jump addr in local .altinstructions */ +static inline bool in_alt_jump(unsigned long jump, void *start, void *end) +{ + return jump >= (unsigned long)start && jump < (unsigned long)end; +} + +static void __init_or_module recompute_jump(union loongarch_instruction *buf, + union loongarch_instruction *dest, union loongarch_instruction *src, + void *start, void *end) +{ + unsigned int si, si_l, si_h; + unsigned long cur_pc, jump_addr, pc; + long offset; + + cur_pc = (unsigned long)src; + pc = (unsigned long)dest; + + si_l = src->reg0i26_format.simmediate_l; + si_h = src->reg0i26_format.simmediate_h; + switch (src->reg0i26_format.opcode) { + case b_op: + case bl_op: + jump_addr = bs_dest_26(cur_pc, si_h, si_l); + if (in_alt_jump(jump_addr, start, end)) + return; + offset = jump_addr - pc; + BUG_ON(offset < -SZ_128M || offset >= SZ_128M); + offset >>= 2; + buf->reg0i26_format.simmediate_h = offset >> 16; + buf->reg0i26_format.simmediate_l = offset; + return; + } + + si_l = src->reg1i21_format.simmediate_l; + si_h = src->reg1i21_format.simmediate_h; + switch (src->reg1i21_format.opcode) { + case beqz_op: + case bnez_op: + case bceqz_op: + jump_addr = bs_dest_21(cur_pc, si_h, si_l); + if (in_alt_jump(jump_addr, start, end)) + return; + offset = jump_addr - pc; + BUG_ON(offset < -SZ_4M || offset >= SZ_4M); + offset >>= 2; + buf->reg1i21_format.simmediate_h = offset >> 16; + buf->reg1i21_format.simmediate_l = offset; + return; + } + + si = src->reg2i16_format.simmediate; + switch (src->reg2i16_format.opcode) { + case beq_op: + case bne_op: + case blt_op: + case bge_op: + case bltu_op: + case bgeu_op: + jump_addr = bs_dest_16(cur_pc, si); + if (in_alt_jump(jump_addr, start, end)) + return; + offset = jump_addr - pc; + BUG_ON(offset < -SZ_128K || offset >= SZ_128K); + offset >>= 2; + buf->reg2i16_format.simmediate = offset; + return; + } +} + +static int __init_or_module copy_alt_insns(union loongarch_instruction *buf, + union loongarch_instruction *dest, union loongarch_instruction *src, int nr) +{ + int i; + + for (i = 0; i < nr; i++) { + buf[i].word = src[i].word; + + if (is_pc_insn(src[i])) { + pr_err("Not support pcrel instruction at present!"); + return -EINVAL; + } + + if (is_branch_insn(src[i]) && + src[i].reg2i16_format.opcode != jirl_op) { + recompute_jump(&buf[i], &dest[i], &src[i], src, src + nr); + } + } + + return 0; +} + +/* + * text_poke_early - Update instructions on a live kernel at boot time + * + * When you use this code to patch more than one byte of an instruction + * you need to make sure that other CPUs cannot execute this code in parallel. + * Also no thread must be currently preempted in the middle of these + * instructions. And on the local CPU you need to be protected again NMI or MCE + * handlers seeing an inconsistent instruction while you patch. + */ +static void *__init_or_module text_poke_early(union loongarch_instruction *insn, + union loongarch_instruction *buf, unsigned int nr) +{ + int i; + unsigned long flags; + + local_irq_save(flags); + + for (i = 0; i < nr; i++) + insn[i].word = buf[i].word; + + local_irq_restore(flags); + + wbflush(); + flush_icache_range((unsigned long)insn, (unsigned long)(insn + nr)); + + return insn; +} + +/* + * Replace instructions with better alternatives for this CPU type. This runs + * before SMP is initialized to avoid SMP problems with self modifying code. + * This implies that asymmetric systems where APs have less capabilities than + * the boot processor are not handled. Tough. Make sure you disable such + * features by hand. + */ +void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) +{ + struct alt_instr *a; + unsigned int nr_instr, nr_repl, nr_insnbuf; + union loongarch_instruction *instr, *replacement; + union loongarch_instruction insnbuf[MAX_PATCH_SIZE]; + + DPRINTK("alt table %px, -> %px", start, end); + /* + * The scan order should be from start to end. A later scanned + * alternative code can overwrite previously scanned alternative code. + * Some kernel functions (e.g. memcpy, memset, etc) use this order to + * patch code. + * + * So be careful if you want to change the scan order to any other + * order. + */ + for (a = start; a < end; a++) { + nr_insnbuf = 0; + + instr = (void *)&a->instr_offset + a->instr_offset; + replacement = (void *)&a->replace_offset + a->replace_offset; + + BUG_ON(a->instrlen > sizeof(insnbuf)); + BUG_ON(a->instrlen & 0x3); + BUG_ON(a->replacementlen & 0x3); + + nr_instr = a->instrlen / LOONGARCH_INSN_SIZE; + nr_repl = a->replacementlen / LOONGARCH_INSN_SIZE; + + if (!cpu_has(a->feature)) { + DPRINTK("feat not exist: %d, old: (%px len: %d), repl: (%px, len: %d)", + a->feature, instr, a->instrlen, + replacement, a->replacementlen); + + continue; + } + + DPRINTK("feat: %d, old: (%px len: %d), repl: (%px, len: %d)", + a->feature, instr, a->instrlen, + replacement, a->replacementlen); + + DUMP_WORDS(instr, nr_instr, "%px: old_insn: ", instr); + DUMP_WORDS(replacement, nr_repl, "%px: rpl_insn: ", replacement); + + copy_alt_insns(insnbuf, instr, replacement, nr_repl); + nr_insnbuf = nr_repl; + + if (nr_instr > nr_repl) { + add_nops(insnbuf + nr_repl, nr_instr - nr_repl); + nr_insnbuf += nr_instr - nr_repl; + } + DUMP_WORDS(insnbuf, nr_insnbuf, "%px: final_insn: ", instr); + + text_poke_early(instr, insnbuf, nr_insnbuf); + } +} + +void __init alternative_instructions(void) +{ + apply_alternatives(__alt_instructions, __alt_instructions_end); + + alternatives_patched = 1; +} diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c new file mode 100644 index 0000000000000000000000000000000000000000..d13690769500c48d4d7993b58fc5c7a5cab2bdcc --- /dev/null +++ b/arch/loongarch/kernel/asm-offsets.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * asm-offsets.c: Calculate pt_regs and task_struct offsets. + * + * Copyright (C) 2020 Loongson Technologies, Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void output_ptreg_defines(void) +{ + COMMENT("LoongArch pt_regs offsets."); + OFFSET(PT_R0, pt_regs, regs[0]); + OFFSET(PT_R1, pt_regs, regs[1]); + OFFSET(PT_R2, pt_regs, regs[2]); + OFFSET(PT_R3, pt_regs, regs[3]); + OFFSET(PT_R4, pt_regs, regs[4]); + OFFSET(PT_R5, pt_regs, regs[5]); + OFFSET(PT_R6, pt_regs, regs[6]); + OFFSET(PT_R7, pt_regs, regs[7]); + OFFSET(PT_R8, pt_regs, regs[8]); + OFFSET(PT_R9, pt_regs, regs[9]); + OFFSET(PT_R10, pt_regs, regs[10]); + OFFSET(PT_R11, pt_regs, regs[11]); + OFFSET(PT_R12, pt_regs, regs[12]); + OFFSET(PT_R13, pt_regs, regs[13]); + OFFSET(PT_R14, pt_regs, regs[14]); + OFFSET(PT_R15, pt_regs, regs[15]); + OFFSET(PT_R16, pt_regs, regs[16]); + OFFSET(PT_R17, pt_regs, regs[17]); + OFFSET(PT_R18, pt_regs, regs[18]); + OFFSET(PT_R19, pt_regs, regs[19]); + OFFSET(PT_R20, pt_regs, regs[20]); + OFFSET(PT_R21, pt_regs, regs[21]); + OFFSET(PT_R22, pt_regs, regs[22]); + OFFSET(PT_R23, pt_regs, regs[23]); + OFFSET(PT_R24, pt_regs, regs[24]); + OFFSET(PT_R25, pt_regs, regs[25]); + OFFSET(PT_R26, pt_regs, regs[26]); + OFFSET(PT_R27, pt_regs, regs[27]); + OFFSET(PT_R28, pt_regs, regs[28]); + OFFSET(PT_R29, pt_regs, regs[29]); + OFFSET(PT_R30, pt_regs, regs[30]); + OFFSET(PT_R31, pt_regs, regs[31]); + OFFSET(PT_CRMD, pt_regs, csr_crmd); + OFFSET(PT_PRMD, pt_regs, csr_prmd); + OFFSET(PT_EUEN, pt_regs, csr_euen); + OFFSET(PT_ECFG, pt_regs, csr_ecfg); + OFFSET(PT_ESTAT, pt_regs, csr_estat); + OFFSET(PT_ERA, pt_regs, csr_era); + OFFSET(PT_BVADDR, pt_regs, csr_badvaddr); + OFFSET(PT_ORIG_A0, pt_regs, orig_a0); + DEFINE(PT_SIZE, sizeof(struct pt_regs)); + BLANK(); +} + +void output_task_defines(void) +{ + COMMENT("LoongArch task_struct offsets."); + OFFSET(TASK_STATE, task_struct, state); + OFFSET(TASK_THREAD_INFO, task_struct, stack); + OFFSET(TASK_FLAGS, task_struct, flags); + OFFSET(TASK_MM, task_struct, mm); + OFFSET(TASK_PID, task_struct, pid); +#if defined(CONFIG_STACKPROTECTOR) + OFFSET(TASK_STACK_CANARY, task_struct, stack_canary); +#endif + DEFINE(TASK_STRUCT_SIZE, sizeof(struct task_struct)); + BLANK(); +} + +void output_thread_info_defines(void) +{ + COMMENT("LoongArch thread_info offsets."); + OFFSET(TI_TASK, thread_info, task); + OFFSET(TI_FLAGS, thread_info, flags); + OFFSET(TI_TP_VALUE, thread_info, tp_value); + OFFSET(TI_CPU, thread_info, cpu); + OFFSET(TI_PRE_COUNT, thread_info, preempt_count); + OFFSET(TI_REGS, thread_info, regs); + DEFINE(_THREAD_SIZE, THREAD_SIZE); + DEFINE(_THREAD_MASK, THREAD_MASK); + DEFINE(_IRQ_STACK_SIZE, IRQ_STACK_SIZE); + DEFINE(_IRQ_STACK_START, IRQ_STACK_START); + BLANK(); +} + +void output_thread_defines(void) +{ + COMMENT("LoongArch specific thread_struct offsets."); + OFFSET(THREAD_REG01, task_struct, thread.reg01); + OFFSET(THREAD_REG03, task_struct, thread.reg03); + OFFSET(THREAD_REG22, task_struct, thread.reg22); + OFFSET(THREAD_REG23, task_struct, thread.reg23); + OFFSET(THREAD_REG24, task_struct, thread.reg24); + OFFSET(THREAD_REG25, task_struct, thread.reg25); + OFFSET(THREAD_REG26, task_struct, thread.reg26); + OFFSET(THREAD_REG27, task_struct, thread.reg27); + OFFSET(THREAD_REG28, task_struct, thread.reg28); + OFFSET(THREAD_REG29, task_struct, thread.reg29); + OFFSET(THREAD_REG30, task_struct, thread.reg30); + OFFSET(THREAD_REG31, task_struct, thread.reg31); + OFFSET(THREAD_SCHED_RA, task_struct, thread.sched_ra); + OFFSET(THREAD_SCHED_CFA, task_struct, thread.sched_cfa); + OFFSET(THREAD_CSRCRMD, task_struct, + thread.csr_crmd); + OFFSET(THREAD_CSRPRMD, task_struct, + thread.csr_prmd); + OFFSET(THREAD_CSREUEN, task_struct, + thread.csr_euen); + OFFSET(THREAD_CSRECFG, task_struct, + thread.csr_ecfg); + + OFFSET(THREAD_FPU, task_struct, thread.fpu); + OFFSET(THREAD_BVADDR, task_struct, \ + thread.csr_badvaddr); + OFFSET(THREAD_ECODE, task_struct, \ + thread.error_code); + OFFSET(THREAD_TRAPNO, task_struct, thread.trap_nr); + BLANK(); +} + +void output_thread_fpu_defines(void) +{ + OFFSET(THREAD_FPR0, loongarch_fpu, fpr[0]); + OFFSET(THREAD_FPR1, loongarch_fpu, fpr[1]); + OFFSET(THREAD_FPR2, loongarch_fpu, fpr[2]); + OFFSET(THREAD_FPR3, loongarch_fpu, fpr[3]); + OFFSET(THREAD_FPR4, loongarch_fpu, fpr[4]); + OFFSET(THREAD_FPR5, loongarch_fpu, fpr[5]); + OFFSET(THREAD_FPR6, loongarch_fpu, fpr[6]); + OFFSET(THREAD_FPR7, loongarch_fpu, fpr[7]); + OFFSET(THREAD_FPR8, loongarch_fpu, fpr[8]); + OFFSET(THREAD_FPR9, loongarch_fpu, fpr[9]); + OFFSET(THREAD_FPR10, loongarch_fpu, fpr[10]); + OFFSET(THREAD_FPR11, loongarch_fpu, fpr[11]); + OFFSET(THREAD_FPR12, loongarch_fpu, fpr[12]); + OFFSET(THREAD_FPR13, loongarch_fpu, fpr[13]); + OFFSET(THREAD_FPR14, loongarch_fpu, fpr[14]); + OFFSET(THREAD_FPR15, loongarch_fpu, fpr[15]); + OFFSET(THREAD_FPR16, loongarch_fpu, fpr[16]); + OFFSET(THREAD_FPR17, loongarch_fpu, fpr[17]); + OFFSET(THREAD_FPR18, loongarch_fpu, fpr[18]); + OFFSET(THREAD_FPR19, loongarch_fpu, fpr[19]); + OFFSET(THREAD_FPR20, loongarch_fpu, fpr[20]); + OFFSET(THREAD_FPR21, loongarch_fpu, fpr[21]); + OFFSET(THREAD_FPR22, loongarch_fpu, fpr[22]); + OFFSET(THREAD_FPR23, loongarch_fpu, fpr[23]); + OFFSET(THREAD_FPR24, loongarch_fpu, fpr[24]); + OFFSET(THREAD_FPR25, loongarch_fpu, fpr[25]); + OFFSET(THREAD_FPR26, loongarch_fpu, fpr[26]); + OFFSET(THREAD_FPR27, loongarch_fpu, fpr[27]); + OFFSET(THREAD_FPR28, loongarch_fpu, fpr[28]); + OFFSET(THREAD_FPR29, loongarch_fpu, fpr[29]); + OFFSET(THREAD_FPR30, loongarch_fpu, fpr[30]); + OFFSET(THREAD_FPR31, loongarch_fpu, fpr[31]); + + OFFSET(THREAD_FCSR, loongarch_fpu, fcsr); + OFFSET(THREAD_FCC, loongarch_fpu, fcc); + OFFSET(THREAD_FTOP, loongarch_fpu, ftop); + BLANK(); +} + +void output_thread_lbt_defines(void) +{ + OFFSET(THREAD_SCR0, loongarch_lbt, scr0); + OFFSET(THREAD_SCR1, loongarch_lbt, scr1); + OFFSET(THREAD_SCR2, loongarch_lbt, scr2); + OFFSET(THREAD_SCR3, loongarch_lbt, scr3); + OFFSET(THREAD_EFLAGS, loongarch_lbt, eflags); + BLANK(); +} + +void output_mm_defines(void) +{ + COMMENT("Size of struct page"); + DEFINE(STRUCT_PAGE_SIZE, sizeof(struct page)); + BLANK(); + COMMENT("Linux mm_struct offsets."); + OFFSET(MM_USERS, mm_struct, mm_users); + OFFSET(MM_PGD, mm_struct, pgd); + OFFSET(MM_CONTEXT, mm_struct, context); + BLANK(); + DEFINE(_PGD_T_SIZE, sizeof(pgd_t)); + DEFINE(_PMD_T_SIZE, sizeof(pmd_t)); + DEFINE(_PTE_T_SIZE, sizeof(pte_t)); + BLANK(); + DEFINE(_PGD_T_LOG2, PGD_T_LOG2); +#ifndef __PAGETABLE_PMD_FOLDED + DEFINE(_PMD_T_LOG2, PMD_T_LOG2); +#endif + DEFINE(_PTE_T_LOG2, PTE_T_LOG2); + BLANK(); + DEFINE(_PGD_ORDER, PGD_ORDER); +#ifndef __PAGETABLE_PMD_FOLDED + DEFINE(_PMD_ORDER, PMD_ORDER); +#endif + DEFINE(_PTE_ORDER, PTE_ORDER); + BLANK(); + DEFINE(_PMD_SHIFT, PMD_SHIFT); + DEFINE(_PGDIR_SHIFT, PGDIR_SHIFT); + BLANK(); + DEFINE(_PTRS_PER_PGD, PTRS_PER_PGD); + DEFINE(_PTRS_PER_PMD, PTRS_PER_PMD); + DEFINE(_PTRS_PER_PTE, PTRS_PER_PTE); + BLANK(); + DEFINE(_PAGE_SHIFT, PAGE_SHIFT); + DEFINE(_PAGE_SIZE, PAGE_SIZE); + BLANK(); +} + +void output_sc_defines(void) +{ + COMMENT("Linux sigcontext offsets."); + OFFSET(SC_REGS, sigcontext, sc_regs); + OFFSET(SC_PC, sigcontext, sc_pc); + BLANK(); +} + +void output_signal_defines(void) +{ + COMMENT("Linux signal numbers."); + DEFINE(_SIGHUP, SIGHUP); + DEFINE(_SIGINT, SIGINT); + DEFINE(_SIGQUIT, SIGQUIT); + DEFINE(_SIGILL, SIGILL); + DEFINE(_SIGTRAP, SIGTRAP); + DEFINE(_SIGIOT, SIGIOT); + DEFINE(_SIGABRT, SIGABRT); + DEFINE(_SIGFPE, SIGFPE); + DEFINE(_SIGKILL, SIGKILL); + DEFINE(_SIGBUS, SIGBUS); + DEFINE(_SIGSEGV, SIGSEGV); + DEFINE(_SIGSYS, SIGSYS); + DEFINE(_SIGPIPE, SIGPIPE); + DEFINE(_SIGALRM, SIGALRM); + DEFINE(_SIGTERM, SIGTERM); + DEFINE(_SIGUSR1, SIGUSR1); + DEFINE(_SIGUSR2, SIGUSR2); + DEFINE(_SIGCHLD, SIGCHLD); + DEFINE(_SIGPWR, SIGPWR); + DEFINE(_SIGWINCH, SIGWINCH); + DEFINE(_SIGURG, SIGURG); + DEFINE(_SIGIO, SIGIO); + DEFINE(_SIGSTOP, SIGSTOP); + DEFINE(_SIGTSTP, SIGTSTP); + DEFINE(_SIGCONT, SIGCONT); + DEFINE(_SIGTTIN, SIGTTIN); + DEFINE(_SIGTTOU, SIGTTOU); + DEFINE(_SIGVTALRM, SIGVTALRM); + DEFINE(_SIGPROF, SIGPROF); + DEFINE(_SIGXCPU, SIGXCPU); + DEFINE(_SIGXFSZ, SIGXFSZ); + BLANK(); +} + +#ifdef CONFIG_SMP +void output_smpboot_defines(void) +{ + COMMENT("Linux smp cpu boot offsets."); + OFFSET(CPU_BOOT_STACK, secondary_data, stack); + OFFSET(CPU_BOOT_TINFO, secondary_data, thread_info); + BLANK(); +} +#endif + +#ifdef CONFIG_HIBERNATION +void output_pbe_defines(void) +{ + COMMENT(" Linux struct pbe offsets. "); + OFFSET(PBE_ADDRESS, pbe, address); + OFFSET(PBE_ORIG_ADDRESS, pbe, orig_address); + OFFSET(PBE_NEXT, pbe, next); + DEFINE(PBE_SIZE, sizeof(struct pbe)); + BLANK(); +} +#endif + +void output_kvm_defines(void) +{ + COMMENT(" KVM/LoongArch Specific offsets. "); + + OFFSET(VCPU_FCSR0, kvm_vcpu_arch, fpu.fcsr); + OFFSET(VCPU_FCC, kvm_vcpu_arch, fpu.fcc); + BLANK(); + + OFFSET(KVM_VCPU_ARCH, kvm_vcpu, arch); + OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm); + OFFSET(KVM_VCPU_RUN, kvm_vcpu, run); + BLANK(); + + OFFSET(KVM_ARCH_HSTACK, kvm_vcpu_arch, host_stack); + OFFSET(KVM_ARCH_HGP, kvm_vcpu_arch, host_gp); + OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); + OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); + OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); + OFFSET(KVM_ARCH_GPC, kvm_vcpu_arch, pc); + OFFSET(KVM_ARCH_GGPR, kvm_vcpu_arch, gprs); + OFFSET(KVM_ARCH_HESTAT, kvm_vcpu_arch, host_estat); + OFFSET(KVM_ARCH_HBADV, kvm_vcpu_arch, badv); + OFFSET(KVM_ARCH_HBADI, kvm_vcpu_arch, badi); + OFFSET(KVM_ARCH_ISHYPCALL, kvm_vcpu_arch, is_hypcall); + OFFSET(KVM_ARCH_HECFG, kvm_vcpu_arch, host_ecfg); + OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); + OFFSET(KVM_ARCH_HPERCPU, kvm_vcpu_arch, host_percpu); + + OFFSET(KVM_GPGD, kvm, arch.gpa_mm.pgd); + BLANK(); + +} diff --git a/arch/loongarch/kernel/cacheinfo.c b/arch/loongarch/kernel/cacheinfo.c new file mode 100644 index 0000000000000000000000000000000000000000..882cb1944a6b68d739e2f7df7d912b146e9b398c --- /dev/null +++ b/arch/loongarch/kernel/cacheinfo.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * LoongArch cacheinfo support + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include + +int init_cache_level(unsigned int cpu) +{ + int cache_present = current_cpu_data.cache_leaves_present; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + this_cpu_ci->num_levels = + current_cpu_data.cache_leaves[cache_present - 1].level; + this_cpu_ci->num_leaves = cache_present; + + return 0; +} + +static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, + struct cacheinfo *sib_leaf) +{ + return (!(*(unsigned char *)(this_leaf->priv) & CACHE_PRIVATE) + && !(*(unsigned char *)(sib_leaf->priv) & CACHE_PRIVATE)); +} + +static void cache_cpumap_setup(unsigned int cpu) +{ + unsigned int index; + struct cacheinfo *this_leaf, *sib_leaf; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + for (index = 0; index < this_cpu_ci->num_leaves; index++) { + unsigned int i; + + this_leaf = this_cpu_ci->info_list + index; + /* skip if shared_cpu_map is already populated */ + if (!cpumask_empty(&this_leaf->shared_cpu_map)) + continue; + + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + for_each_online_cpu(i) { + struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i); + + if (i == cpu || !sib_cpu_ci->info_list || + (cpu_to_node(i) != cpu_to_node(cpu))) + continue; + + sib_leaf = sib_cpu_ci->info_list + index; + if (cache_leaves_are_shared(this_leaf, sib_leaf)) { + cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); + cpumask_set_cpu(i, &this_leaf->shared_cpu_map); + } + } + } +} + +int populate_cache_leaves(unsigned int cpu) +{ + int i, cache_present = current_cpu_data.cache_leaves_present; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + struct cache_desc *cd, *cdesc = current_cpu_data.cache_leaves; + + for (i = 0; i < cache_present; i++) { + cd = cdesc + i; + + this_leaf->type = cd->type; + this_leaf->level = cd->level; + this_leaf->coherency_line_size = cd->linesz; + this_leaf->number_of_sets = cd->sets; + this_leaf->ways_of_associativity = cd->ways; + this_leaf->size = cd->linesz * cd->sets * cd->ways; + this_leaf->priv = &cd->flags; + this_leaf++; + } + + cache_cpumap_setup(cpu); + this_cpu_ci->cpu_map_populated = true; + + return 0; +} diff --git a/arch/loongarch/kernel/cmpxchg.c b/arch/loongarch/kernel/cmpxchg.c new file mode 100644 index 0000000000000000000000000000000000000000..e58c4c69a0bf55c5f4482546746ca6a5ab7505f0 --- /dev/null +++ b/arch/loongarch/kernel/cmpxchg.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 2017 Imagination Technologies + * Author: Paul Burton + */ + +#include +#include + +unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int size) +{ + u32 old32, mask, temp; + volatile u32 *ptr32; + unsigned int shift; + + /* Check that ptr is naturally aligned */ + WARN_ON((unsigned long)ptr & (size - 1)); + + /* Mask value to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + val &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * exchange within the naturally aligned 4 byte integerthat includes + * it. + */ + shift = (unsigned long)ptr & 0x3; + shift *= BITS_PER_BYTE; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + + asm volatile ( + "1: ll.w %0, %3 \n" + " andn %1, %0, %4 \n" + " or %1, %1, %5 \n" + " sc.w %1, %2 \n" + " beqz %1, 1b \n" + : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) + : "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift) + : "memory"); + + return (old32 & mask) >> shift; +} + +unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, + unsigned long new, unsigned int size) +{ + u32 old32, mask, temp; + volatile u32 *ptr32; + unsigned int shift; + + /* Check that ptr is naturally aligned */ + WARN_ON((unsigned long)ptr & (size - 1)); + + /* Mask inputs to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + old &= mask; + new &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * compare & exchange within the naturally aligned 4 byte integer + * that includes it. + */ + shift = (unsigned long)ptr & 0x3; + shift *= BITS_PER_BYTE; + old <<= shift; + new <<= shift; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + + asm volatile ( + "1: ll.w %0, %3 \n" + " and %1, %0, %4 \n" + " bne %1, %5, 2f \n" + " andn %1, %0, %4 \n" + " or %1, %1, %6 \n" + " sc.w %1, %2 \n" + " beqz %1, 1b \n" + " b 3f \n" + "2: \n" + __WEAK_LLSC_MB + "3: \n" + : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) + : "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new) + : "memory"); + + return (old32 & mask) >> shift; +} diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c new file mode 100644 index 0000000000000000000000000000000000000000..552c2c14030bf7476bc5ee2c5f5532bbe24e2c3c --- /dev/null +++ b/arch/loongarch/kernel/cpu-probe.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Processor capabilities determination functions. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* Hardware capabilities */ +unsigned int elf_hwcap __read_mostly; +EXPORT_SYMBOL_GPL(elf_hwcap); + +/* + * Determine the FCSR mask for FPU hardware. + */ +static inline void cpu_set_fpu_fcsr_mask(struct cpuinfo_loongarch *c) +{ + unsigned long sr, mask, fcsr, fcsr0, fcsr1; + + fcsr = c->fpu_csr0; + mask = FPU_CSR_ALL_X | FPU_CSR_ALL_E | FPU_CSR_ALL_S | FPU_CSR_RM; + + sr = read_csr_euen(); + enable_fpu(); + + fcsr0 = fcsr & mask; + write_fcsr(LOONGARCH_FCSR0, fcsr0); + fcsr0 = read_fcsr(LOONGARCH_FCSR0); + + fcsr1 = fcsr | ~mask; + write_fcsr(LOONGARCH_FCSR0, fcsr1); + fcsr1 = read_fcsr(LOONGARCH_FCSR0); + + write_fcsr(LOONGARCH_FCSR0, fcsr); + + write_csr_euen(sr); + + c->fpu_mask = ~(fcsr0 ^ fcsr1) & ~mask; +} + +static inline void set_elf_platform(int cpu, const char *plat) +{ + if (cpu == 0) + __elf_platform = plat; +} + +/* MAP BASE */ +unsigned long vm_map_base; +EXPORT_SYMBOL(vm_map_base); + +static void cpu_probe_addrbits(struct cpuinfo_loongarch *c) +{ +#ifdef __NEED_ADDRBITS_PROBE + c->pabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_PABITS) >> 4; + c->vabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_VABITS) >> 12; + vm_map_base = 0UL - (1UL << c->vabits); +#endif +} + +static void set_isa(struct cpuinfo_loongarch *c, unsigned int isa) +{ + switch (isa) { + case LOONGARCH_CPU_ISA_LA64: + c->isa_level |= LOONGARCH_CPU_ISA_LA64; + fallthrough; + case LOONGARCH_CPU_ISA_LA32S: + c->isa_level |= LOONGARCH_CPU_ISA_LA32S; + fallthrough; + case LOONGARCH_CPU_ISA_LA32R: + c->isa_level |= LOONGARCH_CPU_ISA_LA32R; + break; + } +} + +static void cpu_probe_common(struct cpuinfo_loongarch *c) +{ + unsigned int config; + unsigned long asid_mask; + + c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | + LOONGARCH_CPU_TLB | LOONGARCH_CPU_VINT; + + elf_hwcap = HWCAP_LOONGARCH_CPUCFG; + + config = read_cpucfg(LOONGARCH_CPUCFG1); + if (config & CPUCFG1_UAL) { + c->options |= LOONGARCH_CPU_UAL; + elf_hwcap |= HWCAP_LOONGARCH_UAL; + } + if (config & CPUCFG1_CRC32) { + c->options |= LOONGARCH_CPU_CRC32; + elf_hwcap |= HWCAP_LOONGARCH_CRC32; + } + + + config = read_cpucfg(LOONGARCH_CPUCFG2); + if (config & CPUCFG2_LAM) { + c->options |= LOONGARCH_CPU_LAM; + elf_hwcap |= HWCAP_LOONGARCH_LAM; + } + if (config & CPUCFG2_FP) { + c->options |= LOONGARCH_CPU_FPU; + elf_hwcap |= HWCAP_LOONGARCH_FPU; + } +#ifdef CONFIG_CPU_HAS_LSX + if (config & CPUCFG2_LSX) { + c->options |= LOONGARCH_CPU_LSX; + elf_hwcap |= HWCAP_LOONGARCH_LSX; + } +#endif +#ifdef CONFIG_CPU_HAS_LASX + if (config & CPUCFG2_LASX) { + c->options |= LOONGARCH_CPU_LASX; + elf_hwcap |= HWCAP_LOONGARCH_LASX; + } +#endif + if (config & CPUCFG2_COMPLEX) { + c->options |= LOONGARCH_CPU_COMPLEX; + elf_hwcap |= HWCAP_LOONGARCH_COMPLEX; + } + if (config & CPUCFG2_CRYPTO) { + c->options |= LOONGARCH_CPU_CRYPTO; + elf_hwcap |= HWCAP_LOONGARCH_CRYPTO; + } + if (config & CPUCFG2_PTW) { + c->options |= LOONGARCH_CPU_PTW; + elf_hwcap |= HWCAP_LOONGARCH_PTW; + } + if (config & CPUCFG2_LVZP) { + c->options |= LOONGARCH_CPU_LVZ; + elf_hwcap |= HWCAP_LOONGARCH_LVZ; + } +#ifdef CONFIG_CPU_HAS_LBT + if (config & CPUCFG2_X86BT) { + c->options |= LOONGARCH_CPU_LBT_X86; + elf_hwcap |= HWCAP_LOONGARCH_LBT_X86; + } + if (config & CPUCFG2_ARMBT) { + c->options |= LOONGARCH_CPU_LBT_ARM; + elf_hwcap |= HWCAP_LOONGARCH_LBT_ARM; + } + if (config & CPUCFG2_MIPSBT) { + c->options |= LOONGARCH_CPU_LBT_MIPS; + elf_hwcap |= HWCAP_LOONGARCH_LBT_MIPS; + } +#endif + + config = read_cpucfg(LOONGARCH_CPUCFG6); + if (config & CPUCFG6_PMP) + c->options |= LOONGARCH_CPU_PMP; + + config = iocsr_read32(LOONGARCH_IOCSR_FEATURES); + if (config & IOCSRF_CSRIPI) + c->options |= LOONGARCH_CPU_CSRIPI; + if (config & IOCSRF_EXTIOI) + c->options |= LOONGARCH_CPU_EXTIOI; + if (config & IOCSRF_FREQSCALE) + c->options |= LOONGARCH_CPU_SCALEFREQ; + if (config & IOCSRF_FLATMODE) + c->options |= LOONGARCH_CPU_FLATMODE; + if (config & IOCSRF_EIODECODE) + c->options |= LOONGARCH_CPU_EIODECODE; + if (config & IOCSRF_VM) + c->options |= LOONGARCH_CPU_HYPERVISOR; + + config = csr_read32(LOONGARCH_CSR_ASID); + config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; + asid_mask = GENMASK(config - 1, 0); + set_cpu_asid_mask(c, asid_mask); + + config = read_csr_prcfg1(); + c->ksave_mask = GENMASK((config & CSR_CONF1_KSNUM) - 1, 0); + c->ksave_mask &= ~(EXC_KSAVE_MASK | PERCPU_KSAVE_MASK | KVM_KSAVE_MASK); + + config = read_csr_prcfg3(); + switch (config & CSR_CONF3_TLBTYPE) { + case 0: + c->tlbsizemtlb = 0; + c->tlbsizestlbsets = 0; + c->tlbsizestlbways = 0; + c->tlbsize = 0; + break; + case 1: + c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1; + c->tlbsizestlbsets = 0; + c->tlbsizestlbways = 0; + c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways; + break; + case 2: + c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1; + c->tlbsizestlbsets = 1 << ((config & CSR_CONF3_STLBIDX) >> CSR_CONF3_STLBIDX_SHIFT); + c->tlbsizestlbways = ((config & CSR_CONF3_STLBWAYS) >> CSR_CONF3_STLBWAYS_SHIFT) + 1; + c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways; + break; + default: + pr_warn("Warning: unknown TLB type\n"); + } + + loongarch_probe_watch_registers(c); +} + +static inline void cpu_probe_guestinfo(struct cpuinfo_loongarch *c) +{ + unsigned long guestinfo; + guestinfo = read_csr_gstat(); + if (guestinfo & CSR_GSTAT_GIDBIT) { + c->options |= LOONGARCH_CPU_GUESTID; + write_csr_gstat(0); + } +} + +static inline void cpu_probe_lvz(struct cpuinfo_loongarch *c) +{ + unsigned long gcfg, gprcfg1; + cpu_probe_guestinfo(c); + + c->guest.options |= LOONGARCH_CPU_FPU; + c->guest.options_dyn |= LOONGARCH_CPU_FPU; + c->guest.options_dyn |= LOONGARCH_CPU_PMP; + + c->guest.ases |= LOONGARCH_CPU_LSX; + c->guest.ases_dyn |= LOONGARCH_CPU_LSX; + gprcfg1 = read_gcsr_prcfg1(); + c->guest.kscratch_mask = GENMASK((gprcfg1 & CSR_CONF1_KSNUM) - 1, 0); + + gcfg = read_csr_gcfg(); + if (gcfg & CSR_GCFG_MATP_GUEST) + c->guest_cfg |= BIT(0); + if (gcfg & CSR_GCFG_MATP_ROOT) + c->guest_cfg |= BIT(1); + if (gcfg & CSR_GCFG_MATP_NEST) + c->guest_cfg |= BIT(2); + if (gcfg & CSR_GCFG_SITP) + c->guest_cfg |= BIT(6); + if (gcfg & CSR_GCFG_TITP) + c->guest_cfg |= BIT(8); + if (gcfg & CSR_GCFG_TOEP) + c->guest_cfg |= BIT(10); + if (gcfg & CSR_GCFG_TOPP) + c->guest_cfg |= BIT(12); + if (gcfg & CSR_GCFG_TORUP) + c->guest_cfg |= BIT(14); + if (gcfg & CSR_GCFG_GCIP_ALL) + c->guest_cfg |= BIT(16); + if (gcfg & CSR_GCFG_GCIP_HIT) + c->guest_cfg |= BIT(17); + if (gcfg & CSR_GCFG_GCIP_SECURE) + c->guest_cfg |= BIT(18); +} + +#define MAX_NAME_LEN 32 +#define VENDOR_OFFSET 0 +#define CPUNAME_OFFSET 9 + +static char cpu_full_name[MAX_NAME_LEN] = " - "; + +static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int cpu) +{ + uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]); + uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]); + + if (!__cpu_full_name[cpu]) + __cpu_full_name[cpu] = cpu_full_name; + + *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR); + *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); + + switch (c->processor_id & PRID_SERIES_MASK) { + case PRID_SERIES_LA132: + c->cputype = CPU_LOONGSON32; + set_isa(c, LOONGARCH_CPU_ISA_LA32S); + __cpu_family[cpu] = "Loongson-32bit"; + pr_info("32-bit Loongson Processor probed (LA132 Core)\n"); + break; + case PRID_SERIES_LA264: + c->cputype = CPU_LOONGSON64; + set_isa(c, LOONGARCH_CPU_ISA_LA64); + __cpu_family[cpu] = "Loongson-64bit"; + pr_info("64-bit Loongson Processor probed (LA264 Core)\n"); + break; + case PRID_SERIES_LA364: + c->cputype = CPU_LOONGSON64; + set_isa(c, LOONGARCH_CPU_ISA_LA64); + __cpu_family[cpu] = "Loongson-64bit"; + pr_info("64-bit Loongson Processor probed (LA364 Core)\n"); + break; + case PRID_SERIES_LA464: + c->cputype = CPU_LOONGSON64; + set_isa(c, LOONGARCH_CPU_ISA_LA64); + __cpu_family[cpu] = "Loongson-64bit"; + pr_info("64-bit Loongson Processor probed (LA464 Core)\n"); + break; + case PRID_SERIES_LA664: + c->cputype = CPU_LOONGSON64; + set_isa(c, LOONGARCH_CPU_ISA_LA64); + __cpu_family[cpu] = "Loongson-64bit"; + pr_info("64-bit Loongson Processor probed (LA664 Core)\n"); + break; + default: /* Default to 64 bit */ + c->cputype = CPU_LOONGSON64; + set_isa(c, LOONGARCH_CPU_ISA_LA64); + __cpu_family[cpu] = "Loongson-64bit"; + pr_info("64-bit Loongson Processor probed (Unknown Core)\n"); + } +} + +#ifdef CONFIG_64BIT +/* For use by uaccess.h */ +u64 __ua_limit; +EXPORT_SYMBOL(__ua_limit); +#endif + +const char *__cpu_family[NR_CPUS]; +const char *__cpu_full_name[NR_CPUS]; +const char *__elf_platform; + +static void cpu_report(void) +{ + struct cpuinfo_loongarch *c = ¤t_cpu_data; + + pr_info("CPU%d revision is: %08x (%s)\n", + smp_processor_id(), c->processor_id, cpu_family_string()); + if (c->options & LOONGARCH_CPU_FPU) + pr_info("FPU%d revision is: %08x\n", smp_processor_id(), c->fpu_vers); +} + +void cpu_probe(void) +{ + unsigned int cpu = smp_processor_id(); + struct cpuinfo_loongarch *c = ¤t_cpu_data; + + /* + * Set a default ELF platform, cpu probe may later + * overwrite it with a more precise value + */ + set_elf_platform(cpu, "loongarch"); + + c->cputype = CPU_UNKNOWN; + c->processor_id = read_cpucfg(LOONGARCH_CPUCFG0); + c->fpu_vers = (read_cpucfg(LOONGARCH_CPUCFG2) & CPUCFG2_FPVERS) >> 3; + + c->fpu_csr0 = FPU_CSR_RN; + c->fpu_mask = FPU_CSR_RSVD; + + cpu_probe_common(c); + + per_cpu_trap_init(cpu); + + switch (c->processor_id & PRID_COMP_MASK) { + case PRID_COMP_LOONGSON: + cpu_probe_loongson(c, cpu); + break; + } + + BUG_ON(!__cpu_family[cpu]); + BUG_ON(c->cputype == CPU_UNKNOWN); + + cpu_probe_addrbits(c); + +#ifdef CONFIG_64BIT + if (cpu == 0) + __ua_limit = ~((1ull << cpu_vabits) - 1); +#endif + if (cpu_has_lvz) + cpu_probe_lvz(c); + + cpu_report(); +} diff --git a/arch/loongarch/kernel/crash_dump.c b/arch/loongarch/kernel/crash_dump.c new file mode 100644 index 0000000000000000000000000000000000000000..7aac7bb7d5af34ecff2f7752ea3c3439e543e7c0 --- /dev/null +++ b/arch/loongarch/kernel/crash_dump.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/* + * copy_oldmem_page - copy one page from "oldmem" + * @pfn: page frame number to be copied + * @buf: target memory address for the copy; this can be in kernel address + * space or user address space (see @userbuf) + * @csize: number of bytes to copy + * @offset: offset in bytes into the page (based on pfn) to begin the copy + * @userbuf: if set, @buf is in user address space, use copy_to_user(), + * otherwise @buf is in kernel address space, use memcpy(). + * + * Copy a page from "oldmem". For this page, there is no pte mapped + * in the current kernel. + */ +ssize_t copy_oldmem_page(unsigned long pfn, char *buf, + size_t csize, unsigned long offset, int userbuf) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB); + if (!vaddr) + return -ENOMEM; + + if (!userbuf) { + memcpy(buf, vaddr + offset, csize); + } else { + if (copy_to_user(buf, vaddr + offset, csize)) { + memunmap(vaddr); + csize = -EFAULT; + } + } + + memunmap(vaddr); + + return csize; +} diff --git a/arch/loongarch/kernel/efi.c b/arch/loongarch/kernel/efi.c new file mode 100644 index 0000000000000000000000000000000000000000..2f1d7ac5dddf5752a7860f79c1232133cd8ed08e --- /dev/null +++ b/arch/loongarch/kernel/efi.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * EFI partition + * + * Just for ACPI here, complete it when implementing EFI runtime. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * + * Jianmin Lv: + * Huacai Chen: + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static unsigned long efi_nr_tables; +static unsigned long efi_config_table; +static unsigned long boot_memmap __initdata = EFI_INVALID_TABLE_ADDR; +static unsigned long screen_info_table __initdata = EFI_INVALID_TABLE_ADDR; + +static efi_system_table_t *efi_systab; +static efi_config_table_type_t arch_tables[] __initdata = { + {LINUX_EFI_BOOT_MEMMAP_GUID, &boot_memmap, "MEMMAP" }, + {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table, "SINFO"}, + {}, +}; + +static void __init init_screen_info(void) +{ + struct screen_info *si; + + if (screen_info_table == EFI_INVALID_TABLE_ADDR) + return; + + si = early_memremap_ro(screen_info_table, sizeof(*si)); + if (!si) { + pr_err("Could not map screen_info config table\n"); + return; + } + screen_info = *si; + memset(si, 0, sizeof(*si)); + early_memunmap(si, sizeof(*si)); + + if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI) + memblock_reserve(screen_info.lfb_base, screen_info.lfb_size); +} + +void __init efi_runtime_init(void) +{ + if (!efi_enabled(EFI_BOOT) || !efi_systab->runtime) + return; + + if (efi_runtime_disabled()) { + pr_info("EFI runtime services will be disabled.\n"); + return; + } + + efi.runtime = (efi_runtime_services_t *)efi_systab->runtime; + efi.runtime_version = (unsigned int)efi.runtime->hdr.revision; + + efi_native_runtime_setup(); + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); +} + +void __init efi_init(void) +{ + int size; + void *config_tables; + struct efi_boot_memmap *tbl; + + if (!efi_system_table) + return; + + efi_systab = (efi_system_table_t *)early_memremap_ro(efi_system_table, sizeof(*efi_systab)); + if (!efi_systab) { + pr_err("Can't find EFI system table.\n"); + return; + } + + efi_systab_report_header(&efi_systab->hdr, efi_systab->fw_vendor); + + set_bit(EFI_64BIT, &efi.flags); + efi_nr_tables = efi_systab->nr_tables; + efi_config_table = (unsigned long)efi_systab->tables; + + size = sizeof(efi_config_table_t); + config_tables = early_memremap(efi_config_table, efi_nr_tables * size); + efi_config_parse_tables(config_tables, efi_systab->nr_tables, arch_tables); + early_memunmap(config_tables, efi_nr_tables * size); + + init_screen_info(); + + if (boot_memmap == EFI_INVALID_TABLE_ADDR) + return; + + tbl = early_memremap_ro(boot_memmap, sizeof(*tbl)); + if (tbl) { + struct efi_memory_map_data data; + + data.phys_map = boot_memmap + sizeof(*tbl); + data.size = tbl->map_size; + data.desc_size = tbl->desc_size; + data.desc_version = tbl->desc_ver; + + if (efi_memmap_init_early(&data) < 0) + panic("Unable to map EFI memory map.\n"); + + early_memunmap(tbl, sizeof(*tbl)); + } + + efi_esrt_init(); +} diff --git a/arch/loongarch/kernel/elf.c b/arch/loongarch/kernel/elf.c new file mode 100644 index 0000000000000000000000000000000000000000..9f4d0a8b1168bf35c72fa8a8ffba38e807a57fa6 --- /dev/null +++ b/arch/loongarch/kernel/elf.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include + +#include +#include + +int arch_elf_pt_proc(void *_ehdr, void *_phdr, struct file *elf, + bool is_interp, struct arch_elf_state *state) +{ + return 0; +} + +int arch_check_elf(void *_ehdr, bool has_interpreter, void *_interp_ehdr, + struct arch_elf_state *state) +{ + return 0; +} diff --git a/arch/loongarch/kernel/entry.S b/arch/loongarch/kernel/entry.S new file mode 100644 index 0000000000000000000000000000000000000000..93d65afe9efa05ddf6f2fdf61e5e680563fe1528 --- /dev/null +++ b/arch/loongarch/kernel/entry.S @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include + + .text + .cfi_sections .debug_frame + .align 5 +SYM_CODE_START(handle_syscall) + csrrd t0, PERCPU_BASE_KS + la.abs t1, kernelsp + add.d t1, t1, t0 + move t2, sp + ld.d sp, t1, 0 + + addi.d sp, sp, -PT_SIZE + cfi_st t2, PT_R3 + cfi_rel_offset sp, PT_R3 + st.d zero, sp, PT_R0 + csrrd t2, LOONGARCH_CSR_PRMD + st.d t2, sp, PT_PRMD + csrrd t2, LOONGARCH_CSR_CRMD + st.d t2, sp, PT_CRMD + csrrd t2, LOONGARCH_CSR_EUEN + st.d t2, sp, PT_EUEN + csrrd t2, LOONGARCH_CSR_ECFG + st.d t2, sp, PT_ECFG + csrrd t2, LOONGARCH_CSR_ESTAT + st.d t2, sp, PT_ESTAT + cfi_st ra, PT_R1 + cfi_st a0, PT_R4 + cfi_st a1, PT_R5 + cfi_st a2, PT_R6 + cfi_st a3, PT_R7 + cfi_st a4, PT_R8 + cfi_st a5, PT_R9 + cfi_st a6, PT_R10 + cfi_st a7, PT_R11 + csrrd ra, LOONGARCH_CSR_ERA + st.d ra, sp, PT_ERA + cfi_rel_offset ra, PT_ERA + + cfi_st tp, PT_R2 + cfi_st u0, PT_R21 + cfi_st fp, PT_R22 + + SAVE_STATIC + +#ifdef CONFIG_KGDB + li.w t1, CSR_CRMD_WE + csrxchg t1, t1, LOONGARCH_CSR_CRMD +#endif + UNWIND_HINT_REGS + + move u0, t0 + li.d tp, ~_THREAD_MASK + and tp, tp, sp + + move a0, sp + bl do_syscall + + RESTORE_ALL_AND_RET +SYM_CODE_END(handle_syscall) + +SYM_CODE_START(ret_from_fork) + UNWIND_HINT_REGS + bl schedule_tail # a0 = struct task_struct *prev + move a0, sp + bl syscall_exit_to_user_mode + RESTORE_STATIC + RESTORE_SOME + RESTORE_SP_AND_RET +SYM_CODE_END(ret_from_fork) + +SYM_CODE_START(ret_from_kernel_thread) + UNWIND_HINT_REGS + bl schedule_tail # a0 = struct task_struct *prev + move a0, s1 + jirl ra, s0, 0 + move a0, sp + bl syscall_exit_to_user_mode + RESTORE_STATIC + RESTORE_SOME + RESTORE_SP_AND_RET +SYM_CODE_END(ret_from_kernel_thread) diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S new file mode 100644 index 0000000000000000000000000000000000000000..3917b6accfc7a9bbdfd33a4e0db22f8622c34b2c --- /dev/null +++ b/arch/loongarch/kernel/fpu.S @@ -0,0 +1,616 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2019 Pei Huang + * Copyright (C) 2019 Lu Zeng + * Copyright (C) 2020 Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FPU_REG_WIDTH 8 +#define LSX_REG_WIDTH 16 +#define LASX_REG_WIDTH 32 + + .macro EX insn, reg, src, offs +.ex\@: \insn \reg, \src, \offs + _asm_extable .ex\@, .L_fault + .endm + + .macro EX_V insn, reg, src, offs + parse_v __insn, \insn + parse_v __offs, \offs + parse_r __src, \src + parse_vr __reg, \reg +.ex\@: + .word __insn << 22 | __offs << 10 | __src << 5 | __reg + _asm_extable .ex\@, .L_fault + .endm + + .macro EX_XV insn, reg, src, offs + parse_v __insn, \insn + parse_v __offs, \offs + parse_r __src, \src + parse_xr __reg, \reg +.ex\@: + .word __insn << 22 | __offs << 10 | __src << 5 | __reg + _asm_extable .ex\@, .L_fault + .endm + + .macro sc_save_fp base + EX fst.d $f0, \base, (0 * FPU_REG_WIDTH) + EX fst.d $f1, \base, (1 * FPU_REG_WIDTH) + EX fst.d $f2, \base, (2 * FPU_REG_WIDTH) + EX fst.d $f3, \base, (3 * FPU_REG_WIDTH) + EX fst.d $f4, \base, (4 * FPU_REG_WIDTH) + EX fst.d $f5, \base, (5 * FPU_REG_WIDTH) + EX fst.d $f6, \base, (6 * FPU_REG_WIDTH) + EX fst.d $f7, \base, (7 * FPU_REG_WIDTH) + EX fst.d $f8, \base, (8 * FPU_REG_WIDTH) + EX fst.d $f9, \base, (9 * FPU_REG_WIDTH) + EX fst.d $f10, \base, (10 * FPU_REG_WIDTH) + EX fst.d $f11, \base, (11 * FPU_REG_WIDTH) + EX fst.d $f12, \base, (12 * FPU_REG_WIDTH) + EX fst.d $f13, \base, (13 * FPU_REG_WIDTH) + EX fst.d $f14, \base, (14 * FPU_REG_WIDTH) + EX fst.d $f15, \base, (15 * FPU_REG_WIDTH) + EX fst.d $f16, \base, (16 * FPU_REG_WIDTH) + EX fst.d $f17, \base, (17 * FPU_REG_WIDTH) + EX fst.d $f18, \base, (18 * FPU_REG_WIDTH) + EX fst.d $f19, \base, (19 * FPU_REG_WIDTH) + EX fst.d $f20, \base, (20 * FPU_REG_WIDTH) + EX fst.d $f21, \base, (21 * FPU_REG_WIDTH) + EX fst.d $f22, \base, (22 * FPU_REG_WIDTH) + EX fst.d $f23, \base, (23 * FPU_REG_WIDTH) + EX fst.d $f24, \base, (24 * FPU_REG_WIDTH) + EX fst.d $f25, \base, (25 * FPU_REG_WIDTH) + EX fst.d $f26, \base, (26 * FPU_REG_WIDTH) + EX fst.d $f27, \base, (27 * FPU_REG_WIDTH) + EX fst.d $f28, \base, (28 * FPU_REG_WIDTH) + EX fst.d $f29, \base, (29 * FPU_REG_WIDTH) + EX fst.d $f30, \base, (30 * FPU_REG_WIDTH) + EX fst.d $f31, \base, (31 * FPU_REG_WIDTH) + .endm + + .macro sc_restore_fp base + EX fld.d $f0, \base, (0 * FPU_REG_WIDTH) + EX fld.d $f1, \base, (1 * FPU_REG_WIDTH) + EX fld.d $f2, \base, (2 * FPU_REG_WIDTH) + EX fld.d $f3, \base, (3 * FPU_REG_WIDTH) + EX fld.d $f4, \base, (4 * FPU_REG_WIDTH) + EX fld.d $f5, \base, (5 * FPU_REG_WIDTH) + EX fld.d $f6, \base, (6 * FPU_REG_WIDTH) + EX fld.d $f7, \base, (7 * FPU_REG_WIDTH) + EX fld.d $f8, \base, (8 * FPU_REG_WIDTH) + EX fld.d $f9, \base, (9 * FPU_REG_WIDTH) + EX fld.d $f10, \base, (10 * FPU_REG_WIDTH) + EX fld.d $f11, \base, (11 * FPU_REG_WIDTH) + EX fld.d $f12, \base, (12 * FPU_REG_WIDTH) + EX fld.d $f13, \base, (13 * FPU_REG_WIDTH) + EX fld.d $f14, \base, (14 * FPU_REG_WIDTH) + EX fld.d $f15, \base, (15 * FPU_REG_WIDTH) + EX fld.d $f16, \base, (16 * FPU_REG_WIDTH) + EX fld.d $f17, \base, (17 * FPU_REG_WIDTH) + EX fld.d $f18, \base, (18 * FPU_REG_WIDTH) + EX fld.d $f19, \base, (19 * FPU_REG_WIDTH) + EX fld.d $f20, \base, (20 * FPU_REG_WIDTH) + EX fld.d $f21, \base, (21 * FPU_REG_WIDTH) + EX fld.d $f22, \base, (22 * FPU_REG_WIDTH) + EX fld.d $f23, \base, (23 * FPU_REG_WIDTH) + EX fld.d $f24, \base, (24 * FPU_REG_WIDTH) + EX fld.d $f25, \base, (25 * FPU_REG_WIDTH) + EX fld.d $f26, \base, (26 * FPU_REG_WIDTH) + EX fld.d $f27, \base, (27 * FPU_REG_WIDTH) + EX fld.d $f28, \base, (28 * FPU_REG_WIDTH) + EX fld.d $f29, \base, (29 * FPU_REG_WIDTH) + EX fld.d $f30, \base, (30 * FPU_REG_WIDTH) + EX fld.d $f31, \base, (31 * FPU_REG_WIDTH) + .endm + + .macro sc_save_fcc base, tmp0, tmp1 + movcf2gr \tmp0, $fcc0 + move \tmp1, \tmp0 + movcf2gr \tmp0, $fcc1 + bstrins.d \tmp1, \tmp0, 15, 8 + movcf2gr \tmp0, $fcc2 + bstrins.d \tmp1, \tmp0, 23, 16 + movcf2gr \tmp0, $fcc3 + bstrins.d \tmp1, \tmp0, 31, 24 + movcf2gr \tmp0, $fcc4 + bstrins.d \tmp1, \tmp0, 39, 32 + movcf2gr \tmp0, $fcc5 + bstrins.d \tmp1, \tmp0, 47, 40 + movcf2gr \tmp0, $fcc6 + bstrins.d \tmp1, \tmp0, 55, 48 + movcf2gr \tmp0, $fcc7 + bstrins.d \tmp1, \tmp0, 63, 56 + EX st.d \tmp1, \base, 0 + .endm + + .macro sc_restore_fcc base, tmp0, tmp1 + EX ld.d \tmp0, \base, 0 + bstrpick.d \tmp1, \tmp0, 7, 0 + movgr2cf $fcc0, \tmp1 + bstrpick.d \tmp1, \tmp0, 15, 8 + movgr2cf $fcc1, \tmp1 + bstrpick.d \tmp1, \tmp0, 23, 16 + movgr2cf $fcc2, \tmp1 + bstrpick.d \tmp1, \tmp0, 31, 24 + movgr2cf $fcc3, \tmp1 + bstrpick.d \tmp1, \tmp0, 39, 32 + movgr2cf $fcc4, \tmp1 + bstrpick.d \tmp1, \tmp0, 47, 40 + movgr2cf $fcc5, \tmp1 + bstrpick.d \tmp1, \tmp0, 55, 48 + movgr2cf $fcc6, \tmp1 + bstrpick.d \tmp1, \tmp0, 63, 56 + movgr2cf $fcc7, \tmp1 + .endm + + .macro sc_save_fcsr base, tmp0 + movfcsr2gr \tmp0, fcsr0 + EX st.w \tmp0, \base, 0 +#if defined(CONFIG_CPU_HAS_LBT) + /* TM bit is always 0 if LBT not supported */ + andi \tmp0, \tmp0, FPU_CSR_TM + beqz \tmp0, 1f + bstrins.d \tmp0, $r0, FPU_CSR_TM_SHIFT, FPU_CSR_TM_SHIFT + movgr2fcsr fcsr0, \tmp0 +1: +#endif + .endm + + .macro sc_restore_fcsr base, tmp0 + EX ld.w \tmp0, \base, 0 + movgr2fcsr fcsr0, \tmp0 + .endm + +#if defined(CONFIG_CPU_HAS_LBT) + .macro sc_save_scr base, tmp0 + movscr2gr \tmp0, $scr0 + EX st.d \tmp0, \base, 0 + movscr2gr \tmp0, $scr1 + EX st.d \tmp0, \base, 8 + movscr2gr \tmp0, $scr2 + EX st.d \tmp0, \base, 16 + movscr2gr \tmp0, $scr3 + EX st.d \tmp0, \base, 24 + .endm + + .macro sc_restore_scr base, tmp0 + EX ld.d \tmp0, \base, 0 + movgr2scr $scr0, \tmp0 + EX ld.d \tmp0, \base, 8 + movgr2scr $scr1, \tmp0 + EX ld.d \tmp0, \base, 16 + movgr2scr $scr2, \tmp0 + EX ld.d \tmp0, \base, 24 + movgr2scr $scr3, \tmp0 + .endm +#endif + .macro sc_save_lsx base + EX_V 0xb1 $vr0, \base, (0 * LSX_REG_WIDTH) + EX_V 0xb1 $vr1, \base, (1 * LSX_REG_WIDTH) + EX_V 0xb1 $vr2, \base, (2 * LSX_REG_WIDTH) + EX_V 0xb1 $vr3, \base, (3 * LSX_REG_WIDTH) + EX_V 0xb1 $vr4, \base, (4 * LSX_REG_WIDTH) + EX_V 0xb1 $vr5, \base, (5 * LSX_REG_WIDTH) + EX_V 0xb1 $vr6, \base, (6 * LSX_REG_WIDTH) + EX_V 0xb1 $vr7, \base, (7 * LSX_REG_WIDTH) + EX_V 0xb1 $vr8, \base, (8 * LSX_REG_WIDTH) + EX_V 0xb1 $vr9, \base, (9 * LSX_REG_WIDTH) + EX_V 0xb1 $vr10, \base, (10 * LSX_REG_WIDTH) + EX_V 0xb1 $vr11, \base, (11 * LSX_REG_WIDTH) + EX_V 0xb1 $vr12, \base, (12 * LSX_REG_WIDTH) + EX_V 0xb1 $vr13, \base, (13 * LSX_REG_WIDTH) + EX_V 0xb1 $vr14, \base, (14 * LSX_REG_WIDTH) + EX_V 0xb1 $vr15, \base, (15 * LSX_REG_WIDTH) + EX_V 0xb1 $vr16, \base, (16 * LSX_REG_WIDTH) + EX_V 0xb1 $vr17, \base, (17 * LSX_REG_WIDTH) + EX_V 0xb1 $vr18, \base, (18 * LSX_REG_WIDTH) + EX_V 0xb1 $vr19, \base, (19 * LSX_REG_WIDTH) + EX_V 0xb1 $vr20, \base, (20 * LSX_REG_WIDTH) + EX_V 0xb1 $vr21, \base, (21 * LSX_REG_WIDTH) + EX_V 0xb1 $vr22, \base, (22 * LSX_REG_WIDTH) + EX_V 0xb1 $vr23, \base, (23 * LSX_REG_WIDTH) + EX_V 0xb1 $vr24, \base, (24 * LSX_REG_WIDTH) + EX_V 0xb1 $vr25, \base, (25 * LSX_REG_WIDTH) + EX_V 0xb1 $vr26, \base, (26 * LSX_REG_WIDTH) + EX_V 0xb1 $vr27, \base, (27 * LSX_REG_WIDTH) + EX_V 0xb1 $vr28, \base, (28 * LSX_REG_WIDTH) + EX_V 0xb1 $vr29, \base, (29 * LSX_REG_WIDTH) + EX_V 0xb1 $vr30, \base, (30 * LSX_REG_WIDTH) + EX_V 0xb1 $vr31, \base, (31 * LSX_REG_WIDTH) + .endm + + .macro sc_restore_lsx base + EX_V 0xb0 $vr0, \base, (0 * LSX_REG_WIDTH) + EX_V 0xb0 $vr1, \base, (1 * LSX_REG_WIDTH) + EX_V 0xb0 $vr2, \base, (2 * LSX_REG_WIDTH) + EX_V 0xb0 $vr3, \base, (3 * LSX_REG_WIDTH) + EX_V 0xb0 $vr4, \base, (4 * LSX_REG_WIDTH) + EX_V 0xb0 $vr5, \base, (5 * LSX_REG_WIDTH) + EX_V 0xb0 $vr6, \base, (6 * LSX_REG_WIDTH) + EX_V 0xb0 $vr7, \base, (7 * LSX_REG_WIDTH) + EX_V 0xb0 $vr8, \base, (8 * LSX_REG_WIDTH) + EX_V 0xb0 $vr9, \base, (9 * LSX_REG_WIDTH) + EX_V 0xb0 $vr10, \base, (10 * LSX_REG_WIDTH) + EX_V 0xb0 $vr11, \base, (11 * LSX_REG_WIDTH) + EX_V 0xb0 $vr12, \base, (12 * LSX_REG_WIDTH) + EX_V 0xb0 $vr13, \base, (13 * LSX_REG_WIDTH) + EX_V 0xb0 $vr14, \base, (14 * LSX_REG_WIDTH) + EX_V 0xb0 $vr15, \base, (15 * LSX_REG_WIDTH) + EX_V 0xb0 $vr16, \base, (16 * LSX_REG_WIDTH) + EX_V 0xb0 $vr17, \base, (17 * LSX_REG_WIDTH) + EX_V 0xb0 $vr18, \base, (18 * LSX_REG_WIDTH) + EX_V 0xb0 $vr19, \base, (19 * LSX_REG_WIDTH) + EX_V 0xb0 $vr20, \base, (20 * LSX_REG_WIDTH) + EX_V 0xb0 $vr21, \base, (21 * LSX_REG_WIDTH) + EX_V 0xb0 $vr22, \base, (22 * LSX_REG_WIDTH) + EX_V 0xb0 $vr23, \base, (23 * LSX_REG_WIDTH) + EX_V 0xb0 $vr24, \base, (24 * LSX_REG_WIDTH) + EX_V 0xb0 $vr25, \base, (25 * LSX_REG_WIDTH) + EX_V 0xb0 $vr26, \base, (26 * LSX_REG_WIDTH) + EX_V 0xb0 $vr27, \base, (27 * LSX_REG_WIDTH) + EX_V 0xb0 $vr28, \base, (28 * LSX_REG_WIDTH) + EX_V 0xb0 $vr29, \base, (29 * LSX_REG_WIDTH) + EX_V 0xb0 $vr30, \base, (30 * LSX_REG_WIDTH) + EX_V 0xb0 $vr31, \base, (31 * LSX_REG_WIDTH) + .endm + + .macro sc_save_lasx base + EX_XV 0xb3 $xr0, \base, (0 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr1, \base, (1 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr2, \base, (2 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr3, \base, (3 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr4, \base, (4 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr5, \base, (5 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr6, \base, (6 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr7, \base, (7 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr8, \base, (8 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr9, \base, (9 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr10, \base, (10 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr11, \base, (11 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr12, \base, (12 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr13, \base, (13 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr14, \base, (14 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr15, \base, (15 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr16, \base, (16 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr17, \base, (17 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr18, \base, (18 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr19, \base, (19 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr20, \base, (20 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr21, \base, (21 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr22, \base, (22 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr23, \base, (23 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr24, \base, (24 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr25, \base, (25 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr26, \base, (26 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr27, \base, (27 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr28, \base, (28 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr29, \base, (29 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr30, \base, (30 * LASX_REG_WIDTH) + EX_XV 0xb3 $xr31, \base, (31 * LASX_REG_WIDTH) + .endm + + .macro sc_restore_lasx base + EX_XV 0xb2 $xr0, \base, (0 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr1, \base, (1 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr2, \base, (2 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr3, \base, (3 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr4, \base, (4 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr5, \base, (5 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr6, \base, (6 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr7, \base, (7 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr8, \base, (8 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr9, \base, (9 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr10, \base, (10 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr11, \base, (11 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr12, \base, (12 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr13, \base, (13 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr14, \base, (14 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr15, \base, (15 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr16, \base, (16 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr17, \base, (17 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr18, \base, (18 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr19, \base, (19 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr20, \base, (20 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr21, \base, (21 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr22, \base, (22 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr23, \base, (23 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr24, \base, (24 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr25, \base, (25 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr26, \base, (26 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr27, \base, (27 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr28, \base, (28 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr29, \base, (29 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr30, \base, (30 * LASX_REG_WIDTH) + EX_XV 0xb2 $xr31, \base, (31 * LASX_REG_WIDTH) + .endm + +/* + * Save a thread's fp context. + */ +SYM_FUNC_START(_save_fp) + /* + * since TM bit of FSCR may afftect fpr0-fp7 + * fcsr save need before FPR save + */ + fpu_save_csr a0 t1 + fpu_save_double a0 t1 # clobbers t1 + fpu_save_cc a0 t1 t2 # clobbers t1, t2 + jirl zero, ra, 0 +SYM_FUNC_END(_save_fp) +EXPORT_SYMBOL(_save_fp) + +/* + * Restore a thread's fp context. + */ +SYM_FUNC_START(_restore_fp) + fpu_restore_double a0 t1 # clobbers t1 + /* + * since TM bit of FSCR may afftect fpr0-fp7 + * fscr restore need be after FPR store + */ + fpu_restore_csr a0 t1 + fpu_restore_cc a0 t1 t2 # clobbers t1, t2 + jirl zero, ra, 0 +SYM_FUNC_END(_restore_fp) + +#ifdef CONFIG_CPU_HAS_LSX + +/* + * Save a thread's LSX vector context. + */ +SYM_FUNC_START(_save_lsx) + lsx_save_all a0 t1 t2 + jirl zero, ra, 0 +SYM_FUNC_END(_save_lsx) +EXPORT_SYMBOL(_save_lsx) + +/* + * Restore a thread's LSX vector context. + */ +SYM_FUNC_START(_restore_lsx) + lsx_restore_all a0 t1 t2 + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lsx) + +SYM_FUNC_START(_save_lsx_upper) + lsx_save_all_upper a0 t0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(_save_lsx_upper) + +SYM_FUNC_START(_restore_lsx_upper) + lsx_restore_all_upper a0 t0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lsx_upper) + +SYM_FUNC_START(_init_lsx_upper) + lsx_init_all_upper t1 + jirl zero, ra, 0 +SYM_FUNC_END(_init_lsx_upper) +#endif + +#ifdef CONFIG_CPU_HAS_LASX + +/* + * Save a thread's LASX vector context. + */ +SYM_FUNC_START(_save_lasx) + lasx_save_all a0 t1 t2 + jirl zero, ra, 0 +SYM_FUNC_END(_save_lasx) +EXPORT_SYMBOL(_save_lasx) + +/* + * Restore a thread's LASX vector context. + */ +SYM_FUNC_START(_restore_lasx) + lasx_restore_all a0 t1 t2 + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lasx) + +SYM_FUNC_START(_save_lasx_upper) + lasx_save_all_upper a0 t0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(_save_lasx_upper) + +SYM_FUNC_START(_restore_lasx_upper) + lasx_restore_all_upper a0 t0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lasx_upper) + +SYM_FUNC_START(_init_lasx_upper) + lasx_init_all_upper t1 + jirl zero, ra, 0 +SYM_FUNC_END(_init_lasx_upper) +#endif + +/* + * Load the FPU with signalling NANS. This bit pattern we're using has + * the property that no matter whether considered as single or as double + * precision represents signaling NANS. + * + * The value to initialize fcsr0 to comes in $a0. + */ + +SYM_FUNC_START(_init_fpu) + li.w t1, CSR_EUEN_FPEN + csrxchg t1, t1, LOONGARCH_CSR_EUEN + + movgr2fcsr fcsr0, a0 + + li.w t1, -1 # SNaN + + movgr2fr.d $f0, t1 + movgr2fr.d $f1, t1 + movgr2fr.d $f2, t1 + movgr2fr.d $f3, t1 + movgr2fr.d $f4, t1 + movgr2fr.d $f5, t1 + movgr2fr.d $f6, t1 + movgr2fr.d $f7, t1 + movgr2fr.d $f8, t1 + movgr2fr.d $f9, t1 + movgr2fr.d $f10, t1 + movgr2fr.d $f11, t1 + movgr2fr.d $f12, t1 + movgr2fr.d $f13, t1 + movgr2fr.d $f14, t1 + movgr2fr.d $f15, t1 + movgr2fr.d $f16, t1 + movgr2fr.d $f17, t1 + movgr2fr.d $f18, t1 + movgr2fr.d $f19, t1 + movgr2fr.d $f20, t1 + movgr2fr.d $f21, t1 + movgr2fr.d $f22, t1 + movgr2fr.d $f23, t1 + movgr2fr.d $f24, t1 + movgr2fr.d $f25, t1 + movgr2fr.d $f26, t1 + movgr2fr.d $f27, t1 + movgr2fr.d $f28, t1 + movgr2fr.d $f29, t1 + movgr2fr.d $f30, t1 + movgr2fr.d $f31, t1 + + jirl zero, ra, 0 +SYM_FUNC_END(_init_fpu) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_save_fp_context) + sc_save_fcc a1 t1 t2 + sc_save_fcsr a2 t1 + sc_save_fp a0 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_save_fp_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_restore_fp_context) + sc_restore_fp a0 + sc_restore_fcc a1 t1 t2 + sc_restore_fcsr a2 t1 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_restore_fp_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_save_lsx_context) + sc_save_fcc a1, t0, t1 + sc_save_fcsr a2, t0 + sc_save_lsx a0 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_save_lsx_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_restore_lsx_context) + sc_restore_lsx a0 + sc_restore_fcc a1, t1, t2 + sc_restore_fcsr a2, t1 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lsx_context) + +#if defined(CONFIG_CPU_HAS_LBT) +/* + * Load scr/eflag with zero. + */ +SYM_FUNC_START(_init_lbt) + movgr2scr $scr0, zero + movgr2scr $scr1, zero + movgr2scr $scr2, zero + movgr2scr $scr3, zero + + x86mtflag zero, 0x3f + jr ra +SYM_FUNC_END(_init_lbt) + +/* + * a0: scr + */ +SYM_FUNC_START(_save_scr_context) + /* eflags */ + x86mfflag t0, 0x3f + EX st.w t0, a1, 0 + + sc_save_scr a0, t0 + + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_save_scr_context) + +/* + * a0: scr + */ +SYM_FUNC_START(_restore_scr_context) + /* eflags */ + EX ld.w t0, a1, 0 + x86mtflag t0, 0x3f + + sc_restore_scr a0, t1 + + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_restore_scr_context) +#endif + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_save_lasx_context) + sc_save_fcc a1, t0, t1 + sc_save_fcsr a2, t0 + sc_save_lasx a0 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_save_lasx_context) + +/* + * a0: fpregs + * a1: fcc + * a2: fcsr + */ +SYM_FUNC_START(_restore_lasx_context) + sc_restore_lasx a0 + sc_restore_fcc a1, t1, t2 + sc_restore_fcsr a2, t1 + li.w a0, 0 # success + jirl zero, ra, 0 +SYM_FUNC_END(_restore_lasx_context) + +.L_fault: + li.w a0, -EFAULT # failure + jirl zero, ra, 0 diff --git a/arch/loongarch/kernel/ftrace.c b/arch/loongarch/kernel/ftrace.c new file mode 100644 index 0000000000000000000000000000000000000000..ce125881033d333feb340a84aa41abef40f9634d --- /dev/null +++ b/arch/loongarch/kernel/ftrace.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Code for replacing ftrace calls with jumps. + * + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * Thanks goes to Steven Rostedt for writing the original x86 version. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + +/* + * As `call _mcount` follows LoongArch psABI, ra-saved operation and + * stack operation can be found before this insn. + */ +static inline bool is_ra_save_ins(union loongarch_instruction *ip) +{ + /* st.d $ra, $sp, offset */ + return ip->reg2i12_format.opcode == std_op && + ip->reg2i12_format.rj == 3 && + ip->reg2i12_format.rd == 1; +} + +static inline bool is_stack_open_ins(union loongarch_instruction *ip) +{ + /* addi.d $sp, $sp, -imm */ + return ip->reg2i12_format.opcode == addid_op && + ip->reg2i12_format.rj == 3 && + ip->reg2i12_format.rd == 3 && + ip->reg2i12_format.simmediate < 0; +} + +static int ftrace_get_parent_ra_addr(unsigned long insn_addr, int *ra_off) +{ + union loongarch_instruction *insn; + int limit = 32; + + insn = (union loongarch_instruction *)insn_addr; + + do { + insn--; + limit--; + + if (is_ra_save_ins(insn)) + *ra_off = insn->reg2i12_format.simmediate; + + } while (!is_stack_open_ins(insn) && limit); + + if (!limit) + return -EINVAL; + + return 0; +} + +void prepare_ftrace_return(unsigned long self_addr, + unsigned long callsite_sp, unsigned long old) +{ + int ra_off; + unsigned long return_hooker = (unsigned long)&return_to_handler; + + if (unlikely(ftrace_graph_is_dead())) + return; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + if (ftrace_get_parent_ra_addr(self_addr, &ra_off)) + goto out; + + if (!function_graph_enter(old, self_addr, 0, NULL)) { + *(unsigned long *)(callsite_sp + ra_off) = return_hooker; + } + + return; + +out: + ftrace_graph_stop(); + WARN_ON(1); + return; +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c new file mode 100644 index 0000000000000000000000000000000000000000..4dadfa28f2955348c185233c2fe975b0c0c41c44 --- /dev/null +++ b/arch/loongarch/kernel/ftrace_dyn.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Based on arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2013 Linaro Limited + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include + +#include +#include + +static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, + bool validate) +{ + u32 replaced; + + if (validate) { + if (larch_insn_read((void *)pc, &replaced)) + return -EFAULT; + + if (replaced != old) + return -EINVAL; + } + + if (larch_insn_patch_text((void *)pc, new)) + return -EPERM; + + return 0; +} + +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + unsigned long pc; + u32 new; + + pc = (unsigned long)&ftrace_call; + new = larch_insn_gen_bl(pc, (unsigned long)func); + + return ftrace_modify_code(pc, 0, new, false); +} + +/* + * The compiler has inserted 2 NOPs before the regular function prologue. + * T series registers are available and safe because of LoongArch psABI. + * + * At runtime, replace nop with bl to enable ftrace call and replace bl with + * nop to disable ftrace call. The bl requires us to save the original RA value, + * so here it saves RA at t0. + * details are: + * + * | Compiled | Disabled | Enabled | + * +------------+------------------------+------------------------+ + * | nop | move t0, ra | move t0, ra | + * | nop | nop | bl ftrace_caller | + * | func_body | func_body | func_body | + * + * The RA value will be recovered by ftrace_regs_entry, and restored into RA + * before returning to the regular function prologue. When a function is not + * being traced, the move t0, ra is not harmful. + */ + +int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) +{ + unsigned long pc; + u32 old, new; + + pc = rec->ip; + old = larch_insn_gen_nop(); + new = larch_insn_gen_move(LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); + + return ftrace_modify_code(pc, old, new, true); +} + +static inline int __get_mod(struct module **mod, unsigned long addr) +{ + preempt_disable(); + *mod = __module_text_address(addr); + preempt_enable(); + + if (WARN_ON(!(*mod))) + return -EINVAL; + + return 0; +} + +static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr) +{ + struct plt_entry *plt = mod->arch.ftrace_trampolines; + + if (addr == FTRACE_ADDR) + return &plt[FTRACE_PLT_IDX]; + if (addr == FTRACE_REGS_ADDR && + IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) + return &plt[FTRACE_REGS_PLT_IDX]; + + return NULL; +} + +static unsigned long get_plt_addr(struct module *mod, unsigned long addr) +{ + struct plt_entry *plt; + + plt = get_ftrace_plt(mod, addr); + if (!plt) { + pr_err("ftrace: no module PLT for %ps\n", (void *)addr); + return -EINVAL; + } + + return (unsigned long)plt; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long pc; + long offset; + u32 old, new; + + pc = rec->ip + LOONGARCH_INSN_SIZE; + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + } + + old = larch_insn_gen_nop(); + new = larch_insn_gen_bl(pc, addr); + + return ftrace_modify_code(pc, old, new, true); +} + +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, + unsigned long addr) +{ + unsigned long pc; + long offset; + u32 old, new; + + pc = rec->ip + LOONGARCH_INSN_SIZE; + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + } + + new = larch_insn_gen_nop(); + old = larch_insn_gen_bl(pc, addr); + + return ftrace_modify_code(pc, old, new, true); +} + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) +{ + unsigned long pc; + long offset; + u32 old, new; + + pc = rec->ip + LOONGARCH_INSN_SIZE; + offset = (long)pc - (long)addr; + + if (offset < -SZ_128M || offset >= SZ_128M) { + int ret; + struct module *mod; + + ret = __get_mod(&mod, pc); + if (ret) + return ret; + + addr = get_plt_addr(mod, addr); + + old_addr = get_plt_addr(mod, old_addr); + } + + old = larch_insn_gen_bl(pc, old_addr); + new = larch_insn_gen_bl(pc, addr); + + return ftrace_modify_code(pc, old, new, true); +} +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ + +void arch_ftrace_update_code(int command) +{ + command |= FTRACE_MAY_SLEEP; + ftrace_modify_all_code(command); +} + +int __init ftrace_dyn_arch_init(void) +{ + return 0; +} + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +extern void ftrace_graph_call(void); + +void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + old = *parent; + + if (!function_graph_enter(old, self_addr, 0, parent)) + *parent = return_hooker; +} + +static int ftrace_modify_graph_caller(bool enable) +{ + unsigned long pc, func; + u32 branch, nop; + + pc = (unsigned long)&ftrace_graph_call; + func = (unsigned long)&ftrace_graph_caller; + + branch = larch_insn_gen_b(pc, func); + nop = larch_insn_gen_nop(); + + if (enable) + return ftrace_modify_code(pc, nop, branch, true); + else + return ftrace_modify_code(pc, branch, nop, true); +} + +int ftrace_enable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(false); +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + +#ifdef CONFIG_KPROBES_ON_FTRACE +/* Ftrace callback handler for kprobes -- called under preepmt disabed */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct pt_regs *regs) +{ + struct kprobe *p; + struct kprobe_ctlblk *kcb; + + p = get_kprobe((kprobe_opcode_t *)ip); + if (unlikely(!p) || kprobe_disabled(p)) + return; + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + unsigned long orig_ip = regs->csr_era; + + regs->csr_era = ip; + + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, regs)) { + /* + * Emulate singlestep (and also recover regs->csr_era) + * as if there is a nop. + */ + regs->csr_era = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + regs->csr_era = orig_ip; + } + /* + * If pre_handler returns !0, it changes regs->ip. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); + } +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.insn = NULL; + return 0; +} +#endif /* CONFIG_KPROBES_ON_FTRACE */ diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S new file mode 100644 index 0000000000000000000000000000000000000000..828f3a369f9e29448ca76eca4f7d73186c3115be --- /dev/null +++ b/arch/loongarch/kernel/genex.S @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include + + .align 5 +SYM_FUNC_START(__arch_cpu_idle) + /* start of rollback region */ + LONG_L t0, tp, TI_FLAGS + nop + andi t0, t0, _TIF_NEED_RESCHED + bnez t0, 1f + nop + nop + nop + idle 0 + /* end of rollback region */ +1: jirl zero, ra, 0 +SYM_FUNC_END(__arch_cpu_idle) + +SYM_CODE_START(handle_vint) + BACKUP_T0T1 + SAVE_ALL + UNWIND_HINT_REGS + la.abs t1, __arch_cpu_idle + LONG_L t0, sp, PT_ERA + /* 32 byte rollback region */ + ori t0, t0, 0x1f + xori t0, t0, 0x1f + bne t0, t1, 1f + LONG_S t0, sp, PT_ERA +1: move a0, sp + move a1, sp + la.abs t0, do_vint + jirl ra, t0, 0 + UNWIND_HINT_REGS + RESTORE_ALL_AND_RET +SYM_CODE_END(handle_vint) + +SYM_CODE_START(except_vec_cex) + b cache_parity_error +SYM_CODE_END(except_vec_cex) + + .macro build_prep_badv + csrrd t0, LOONGARCH_CSR_BADV + PTR_S t0, sp, PT_BVADDR + .endm + + .macro build_prep_fcsr + movfcsr2gr a1, fcsr0 + .endm + + .macro build_prep_none + .endm + + .macro BUILD_HANDLER exception handler prep + .align 5 + SYM_CODE_START(handle_\exception) + BACKUP_T0T1 + SAVE_ALL + build_prep_\prep + move a0, sp + la.abs t0, do_\handler + jirl ra, t0, 0 + UNWIND_HINT_REGS + RESTORE_ALL_AND_RET + SYM_CODE_END(handle_\exception) + .endm + + BUILD_HANDLER ade ade badv + BUILD_HANDLER ale ale badv + BUILD_HANDLER bce bce none + BUILD_HANDLER bp bp none + BUILD_HANDLER fpe fpe fcsr + BUILD_HANDLER fpu fpu none + BUILD_HANDLER lsx lsx none + BUILD_HANDLER lasx lasx none + BUILD_HANDLER lbt lbt none + BUILD_HANDLER ri ri none + BUILD_HANDLER watch watch none + BUILD_HANDLER reserved reserved none /* others */ + +SYM_CODE_START(handle_sys) + la.abs t0, handle_syscall + jirl zero, t0, 0 +SYM_CODE_END(handle_sys) diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S new file mode 100644 index 0000000000000000000000000000000000000000..e0742de8ae413fb7acbdd9b822a22498db1c0384 --- /dev/null +++ b/arch/loongarch/kernel/head.S @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +SYM_ENTRY(_stext, SYM_L_GLOBAL, SYM_A_NONE) + + __REF + + .align 12 + +SYM_CODE_START(kernel_entry) # kernel entry point + + /* Config direct window and set PG */ + li.d t0, CSR_DMW0_INIT # UC, PLV0, 0x8000 xxxx xxxx xxxx + csrwr t0, LOONGARCH_CSR_DMWIN0 + li.d t0, CSR_DMW1_INIT # CA, PLV0, 0x9000 xxxx xxxx xxxx + csrwr t0, LOONGARCH_CSR_DMWIN1 + + JUMP_VIRT_ADDR t0, t1 + + /* Enable PG */ + li.w t0, 0xb0 # PLV=0, IE=0, PG=1 + csrwr t0, LOONGARCH_CSR_CRMD + li.w t0, 0x04 # PLV=0, PIE=1, PWE=0 + csrwr t0, LOONGARCH_CSR_PRMD + li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 + csrwr t0, LOONGARCH_CSR_EUEN + + la.pcrel t0, __bss_start # clear .bss + st.d zero, t0, 0 + la.pcrel t1, __bss_stop - LONGSIZE +1: + addi.d t0, t0, LONGSIZE + st.d zero, t0, 0 + bne t0, t1, 1b + + la.pcrel t0, fw_arg0 + st.d a0, t0, 0 # firmware arguments + la.pcrel t0, fw_arg1 + st.d a1, t0, 0 + la.pcrel t0, fw_arg2 + st.d a2, t0, 0 + +#ifdef CONFIG_PAGE_SIZE_4KB + li.d t0, 0 + li.d t1, CSR_STFILL + csrxchg t0, t1, LOONGARCH_CSR_IMPCTL1 +#endif + /* KSave3 used for percpu base, initialized as 0 */ + csrwr zero, PERCPU_BASE_KS + /* GPR21 used for percpu base (runtime), initialized as 0 */ + or u0, zero, zero + + la.pcrel tp, init_thread_union + /* Set the SP after an empty pt_regs. */ + PTR_LI sp, (_THREAD_SIZE - PT_SIZE) + PTR_ADD sp, sp, tp + set_saved_sp sp, t0, t1 + +#ifdef CONFIG_KASAN + bl kasan_early_init +#endif + +#ifdef CONFIG_RELOCATABLE +#ifdef CONFIG_CRASH_DUMP + beqz a3, 1f + move a0, a3 + + /* Apply the relocations for kdump */ + bl relocate_kdump_kernel + b 2f +#endif +1: + /* Copy kernel and apply the relocations */ + bl relocate_kernel + +2: + /* Repoint the sp into the new kernel image */ + PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE) + PTR_ADD sp, sp, tp + set_saved_sp sp, t0, t1 + + /* + * relocate_kernel returns the entry point either + * in the relocated kernel or the original if for + * some reason relocation failed. + */ + jirl zero, a0, 0 +#else + bl start_kernel +#endif + ASM_BUG() + +SYM_CODE_END(kernel_entry) + +#ifdef CONFIG_SMP + +/* + * SMP slave cpus entry point. Board specific code for bootstrap calls this + * function after setting up the stack and tp registers. + */ +SYM_CODE_START(smpboot_entry) + li.d t0, CSR_DMW0_INIT # UC, PLV0 + csrwr t0, LOONGARCH_CSR_DMWIN0 + li.d t0, CSR_DMW1_INIT # CA, PLV0 + csrwr t0, LOONGARCH_CSR_DMWIN1 + + JUMP_VIRT_ADDR t0, t1 + +#ifdef CONFIG_PAGE_SIZE_4KB + li.d t0, 0 + li.d t1, CSR_STFILL + csrxchg t0, t1, LOONGARCH_CSR_IMPCTL1 +#endif + /* Enable PG */ + li.w t0, 0xb0 # PLV=0, IE=0, PG=1 + csrwr t0, LOONGARCH_CSR_CRMD + li.w t0, 0x04 # PLV=0, PIE=1, PWE=0 + csrwr t0, LOONGARCH_CSR_PRMD + li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 + csrwr t0, LOONGARCH_CSR_EUEN + + la.abs t0, cpuboot_data + ld.d sp, t0, CPU_BOOT_STACK + ld.d tp, t0, CPU_BOOT_TINFO + + bl start_secondary + ASM_BUG() + +SYM_CODE_END(smpboot_entry) + +#endif /* CONFIG_SMP */ + +SYM_ENTRY(kernel_entry_end, SYM_L_GLOBAL, SYM_A_NONE) diff --git a/arch/loongarch/kernel/idle.c b/arch/loongarch/kernel/idle.c new file mode 100644 index 0000000000000000000000000000000000000000..27030472ce84610dd1ea73fc739e0fa7888afa76 --- /dev/null +++ b/arch/loongarch/kernel/idle.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LoongArch idle loop support. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +void __cpuidle arch_cpu_idle(void) +{ + raw_local_irq_enable(); + __arch_cpu_idle(); /* idle instruction needs irq enabled */ +} diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c new file mode 100644 index 0000000000000000000000000000000000000000..60fb04c8d6c4d95e1c47d83d68a2057a2ea92368 --- /dev/null +++ b/arch/loongarch/kernel/inst.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include + +#include +#include +#include + +#define __SIGNEX(X, SIDX) ((X) >= (1 << SIDX) ? ~((1 << SIDX) - 1) | (X) : (X)) +#define SIGNEX16(X) __SIGNEX(((unsigned long)(X)), 15) +#define SIGNEX20(X) __SIGNEX(((unsigned long)(X)), 19) +#define SIGNEX21(X) __SIGNEX(((unsigned long)(X)), 20) +#define SIGNEX26(X) __SIGNEX(((unsigned long)(X)), 25) + +unsigned long bs_dest_16(unsigned long now, unsigned int si) +{ + return now + (SIGNEX16(si) << 2); +} + +unsigned long bs_dest_21(unsigned long now, unsigned int h, unsigned int l) +{ + return now + (SIGNEX21(h << 16 | l) << 2); +} + +unsigned long bs_dest_26(unsigned long now, unsigned int h, unsigned int l) +{ + return now + (SIGNEX26(h << 16 | l) << 2); +} + +int simu_branch(struct pt_regs *regs, union loongarch_instruction insn) +{ + unsigned int si, si_l, si_h, rd, rj; + unsigned long era = regs->csr_era; + + if (era & 3) + return -EFAULT; + + si_l = insn.reg0i26_format.simmediate_l; + si_h = insn.reg0i26_format.simmediate_h; + switch (insn.reg0i26_format.opcode) { + case b_op: + regs->csr_era = bs_dest_26(era, si_h, si_l); + return 0; + case bl_op: + regs->csr_era = bs_dest_26(era, si_h, si_l); + regs->regs[1] = era + LOONGARCH_INSN_SIZE; + return 0; + } + + si_l = insn.reg1i21_format.simmediate_l; + si_h = insn.reg1i21_format.simmediate_h; + rj = insn.reg1i21_format.rj; + switch (insn.reg1i21_format.opcode) { + case beqz_op: + if (cond_beqz(regs, rj)) + regs->csr_era = bs_dest_21(era, si_h, si_l); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + return 0; + case bnez_op: + if (cond_bnez(regs, rj)) + regs->csr_era = bs_dest_21(era, si_h, si_l); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + return 0; + } + + si = insn.reg2i16_format.simmediate; + rj = insn.reg2i16_format.rj; + rd = insn.reg2i16_format.rd; + switch (insn.reg2i16_format.opcode) { + case beq_op: + if (cond_beq(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case bne_op: + if (cond_bne(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case blt_op: + if (cond_blt(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case bge_op: + if (cond_bge(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case bltu_op: + if (cond_bltu(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case bgeu_op: + if (cond_bgeu(regs, rj, rd)) + regs->csr_era = bs_dest_16(era, si); + else + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + case jirl_op: + /* Use bs_dest_16() to calc the dest. */ + regs->csr_era = bs_dest_16(regs->regs[rj], si); + regs->regs[rd] = era + LOONGARCH_INSN_SIZE; + break; + default: + return -EINVAL; + } + + return 0; +} + +int simu_pc(struct pt_regs *regs, union loongarch_instruction insn) +{ + unsigned long era = regs->csr_era; + unsigned int rd = insn.reg1i20_format.rd; + unsigned int si = insn.reg1i20_format.simmediate; + + if (era & 3) + return -EFAULT; + + switch (insn.reg1i20_format.opcode) { + case pcaddi_op: + regs->regs[rd] = era + (SIGNEX20(si) << 2); + break; + case pcalau12i_op: + regs->regs[rd] = era + (SIGNEX20(si) << 12); + regs->regs[rd] &= ~((1 << 12) - 1); + break; + case pcaddu12i_op: + regs->regs[rd] = era + (SIGNEX20(si) << 12); + break; + case pcaddu18i_op: + regs->regs[rd] = era + (SIGNEX20(si) << 18); + break; + default: + return -EINVAL; + } + + regs->csr_era += LOONGARCH_INSN_SIZE; + + return 0; +} + +static DEFINE_RAW_SPINLOCK(patch_lock); + +int larch_insn_read(void *addr, u32 *insnp) +{ + int ret; + u32 val; + + ret = copy_from_kernel_nofault(&val, addr, LOONGARCH_INSN_SIZE); + if (!ret) + *insnp = val; + + return ret; +} + +int larch_insn_write(void *addr, u32 insn) +{ + int ret; + unsigned long flags = 0; + + raw_spin_lock_irqsave(&patch_lock, flags); + ret = copy_to_kernel_nofault(addr, &insn, LOONGARCH_INSN_SIZE); + raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + +int larch_insn_patch_text(void *addr, u32 insn) +{ + int ret; + u32 *tp = addr; + + if ((unsigned long)tp & 3) + return -EINVAL; + + ret = larch_insn_write(tp, insn); + if (!ret) + flush_icache_range((unsigned long)tp, + (unsigned long)tp + LOONGARCH_INSN_SIZE); + + return ret; +} + +u32 larch_insn_gen_nop(void) +{ + return INSN_NOP; +} + +u32 larch_insn_gen_b(unsigned long pc, unsigned long dest) +{ + unsigned int simmediate_l, simmediate_h; + union loongarch_instruction insn; + long offset = dest - pc; + + if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) { + pr_warn("The generated b instruction is out of range.\n"); + return INSN_BREAK; + } + + offset >>= 2; + + simmediate_l = offset & 0xffff; + offset >>= 16; + simmediate_h = offset & 0x3ff; + + insn.reg0i26_format.opcode = b_op; + insn.reg0i26_format.simmediate_l = simmediate_l; + insn.reg0i26_format.simmediate_h = simmediate_h; + + return insn.word; +} + +u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest) +{ + unsigned int simmediate_l, simmediate_h; + union loongarch_instruction insn; + long offset = dest - pc; + + if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) { + pr_warn("The generated bl instruction is out of range.\n"); + return INSN_BREAK; + } + + offset >>= 2; + + simmediate_l = offset & 0xffff; + offset >>= 16; + simmediate_h = offset & 0x3ff; + + insn.reg0i26_format.opcode = bl_op; + insn.reg0i26_format.simmediate_l = simmediate_l; + insn.reg0i26_format.simmediate_h = simmediate_h; + + return insn.word; +} + +u32 larch_insn_gen_addu16id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + union loongarch_instruction insn; + + insn.reg2i16_format.opcode = addu16id_op; + insn.reg2i16_format.rd = rd; + insn.reg2i16_format.rj = rj; + insn.reg2i16_format.simmediate = imm; + + return insn.word; +} + +u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm) +{ + union loongarch_instruction insn; + + insn.reg1i20_format.opcode = lu32id_op; + insn.reg1i20_format.rd = rd; + insn.reg1i20_format.simmediate = imm; + + return insn.word; +} + +u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + union loongarch_instruction insn; + + insn.reg2i12_format.opcode = lu52id_op; + insn.reg2i12_format.rd = rd; + insn.reg2i12_format.rj = rj; + insn.reg2i12_format.simmediate = imm; + + return insn.word; +} + +u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, + unsigned long pc, unsigned long dest) +{ + union loongarch_instruction insn; + long offset = dest - pc; + + if ((offset & 3) || offset < -SZ_128K || offset >= SZ_128K) { + pr_warn("The generated jirl instruction is out of range.\n"); + return INSN_BREAK; + } + + insn.reg2i16_format.opcode = jirl_op; + insn.reg2i16_format.rd = rd; + insn.reg2i16_format.rj = rj; + insn.reg2i16_format.simmediate = offset >> 2; + + return insn.word; +} + +u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, + enum loongarch_gpr rk) +{ + union loongarch_instruction insn; + + insn.reg3_format.opcode = or_op; + insn.reg3_format.rd = rd; + insn.reg3_format.rj = rj; + insn.reg3_format.rk = rk; + + return insn.word; +} + +u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj) +{ + return larch_insn_gen_or(rd, rj, 0); +} diff --git a/arch/loongarch/kernel/io.c b/arch/loongarch/kernel/io.c new file mode 100644 index 0000000000000000000000000000000000000000..07c5fb99cbb4a28a2fd4b02757d9dfba22f279c7 --- /dev/null +++ b/arch/loongarch/kernel/io.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include + +/* + * Copy data from IO memory space to "real" memory space. + */ +void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) +{ + while (count && !IS_ALIGNED((unsigned long)from, 8)) { + *(u8 *)to = __raw_readb(from); + from++; + to++; + count--; + } + + while (count >= 8) { + *(u64 *)to = __raw_readq(from); + from += 8; + to += 8; + count -= 8; + } + + while (count) { + *(u8 *)to = __raw_readb(from); + from++; + to++; + count--; + } +} +EXPORT_SYMBOL(__memcpy_fromio); + +/* + * Copy data from "real" memory space to IO memory space. + */ +void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count) +{ + while (count && !IS_ALIGNED((unsigned long)to, 8)) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } + + while (count >= 8) { + __raw_writeq(*(u64 *)from, to); + from += 8; + to += 8; + count -= 8; + } + + while (count) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } +} +EXPORT_SYMBOL(__memcpy_toio); + +/* + * "memset" on IO memory space. + */ +void __memset_io(volatile void __iomem *dst, int c, size_t count) +{ + u64 qc = (u8)c; + + qc |= qc << 8; + qc |= qc << 16; + qc |= qc << 32; + + while (count && !IS_ALIGNED((unsigned long)dst, 8)) { + __raw_writeb(c, dst); + dst++; + count--; + } + + while (count >= 8) { + __raw_writeq(qc, dst); + dst += 8; + count -= 8; + } + + while (count) { + __raw_writeb(c, dst); + dst++; + count--; + } +} +EXPORT_SYMBOL(__memset_io); diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c new file mode 100644 index 0000000000000000000000000000000000000000..0b83d1ea590f91a027e63c11c34fa3e95058bb35 --- /dev/null +++ b/arch/loongarch/kernel/irq.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +DEFINE_PER_CPU(unsigned long, irq_stack); + +/* + * 'what should we do if we get a hw irq event on an illegal vector'. + * each architecture has to answer this themselves. + */ +void ack_bad_irq(unsigned int irq) +{ + printk("unexpected IRQ # %d\n", irq); +} + +atomic_t irq_err_count; + +int arch_show_interrupts(struct seq_file *p, int prec) +{ +#ifdef CONFIG_SMP + show_ipi_list(p, prec); +#endif + seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); + return 0; +} + +asmlinkage void spurious_interrupt(void) +{ + atomic_inc(&irq_err_count); +} + +void __init init_IRQ(void) +{ + int i; + unsigned int order = get_order(IRQ_STACK_SIZE); + struct page *page; + + for (i = 0; i < NR_IRQS; i++) + irq_set_noprobe(i); + + arch_init_irq(); + + for_each_possible_cpu(i) { + page = alloc_pages_node(cpu_to_node(i), GFP_KERNEL, order); + + per_cpu(irq_stack, i) = (unsigned long)page_address(page); + pr_debug("CPU%d IRQ stack at 0x%lx - 0x%lx\n", i, + per_cpu(irq_stack, i), per_cpu(irq_stack, i) + IRQ_STACK_SIZE); + } +} diff --git a/arch/loongarch/kernel/jump_label.c b/arch/loongarch/kernel/jump_label.c new file mode 100644 index 0000000000000000000000000000000000000000..53c2b70aebdbc3838477940afab6865dd18ce3ad --- /dev/null +++ b/arch/loongarch/kernel/jump_label.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Loongson Technology Corporation Limited + * + * Based on arch/arm64/kernel/jump_label.c + */ +#include +#include +#include +#include + +void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) +{ + u32 insn; + void *addr = (void *)jump_entry_code(entry); + + if (type == JUMP_LABEL_JMP) + insn = larch_insn_gen_b(jump_entry_code(entry), jump_entry_target(entry)); + else + insn = larch_insn_gen_nop(); + + larch_insn_patch_text(addr, insn); +} diff --git a/arch/loongarch/kernel/kfpu.c b/arch/loongarch/kernel/kfpu.c new file mode 100644 index 0000000000000000000000000000000000000000..ec5b28e570c963482d18e50f28043b066a425ffc --- /dev/null +++ b/arch/loongarch/kernel/kfpu.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + +static unsigned int euen_mask = CSR_EUEN_FPEN; + +/* + * The critical section between kernel_fpu_begin() and kernel_fpu_end() + * is non-reentrant. It is the caller's responsibility to avoid reentrance. + * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example. + */ +static DEFINE_PER_CPU(bool, in_kernel_fpu); +static DEFINE_PER_CPU(unsigned int, euen_current); + +void kernel_fpu_begin(void) +{ + unsigned int *euen_curr; + + preempt_disable(); + + WARN_ON(this_cpu_read(in_kernel_fpu)); + + this_cpu_write(in_kernel_fpu, true); + euen_curr = this_cpu_ptr(&euen_current); + + *euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN); + +#ifdef CONFIG_CPU_HAS_LASX + if (*euen_curr & CSR_EUEN_LASXEN) + _save_lasx(¤t->thread.fpu); + else +#endif +#ifdef CONFIG_CPU_HAS_LSX + if (*euen_curr & CSR_EUEN_LSXEN) + _save_lsx(¤t->thread.fpu); + else +#endif + if (*euen_curr & CSR_EUEN_FPEN) + _save_fp(¤t->thread.fpu); + + write_fcsr(LOONGARCH_FCSR0, 0); +} +EXPORT_SYMBOL_GPL(kernel_fpu_begin); + +void kernel_fpu_end(void) +{ + unsigned int *euen_curr; + + WARN_ON(!this_cpu_read(in_kernel_fpu)); + + euen_curr = this_cpu_ptr(&euen_current); + +#ifdef CONFIG_CPU_HAS_LASX + if (*euen_curr & CSR_EUEN_LASXEN) + _restore_lasx(¤t->thread.fpu); + else +#endif +#ifdef CONFIG_CPU_HAS_LSX + if (*euen_curr & CSR_EUEN_LSXEN) + _restore_lsx(¤t->thread.fpu); + else +#endif + if (*euen_curr & CSR_EUEN_FPEN) + _restore_fp(¤t->thread.fpu); + + *euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN); + + this_cpu_write(in_kernel_fpu, false); + + preempt_enable(); +} +EXPORT_SYMBOL_GPL(kernel_fpu_end); + +static int __init init_euen_mask(void) +{ + if (cpu_has_lsx) + euen_mask |= CSR_EUEN_LSXEN; + + if (cpu_has_lasx) + euen_mask |= CSR_EUEN_LASXEN; + + return 0; +} +arch_initcall(init_euen_mask); diff --git a/arch/loongarch/kernel/kgdb.c b/arch/loongarch/kernel/kgdb.c new file mode 100644 index 0000000000000000000000000000000000000000..f5f6fe178450bd126e29322dd90cacda926ca452 --- /dev/null +++ b/arch/loongarch/kernel/kgdb.c @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include /* for linux pt_regs struct */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +static int kgdb_watch_dcount; +static int kgdb_watch_icount; +int kgdb_watch_activated; + +int param_set_dcount(const char *val, const struct kernel_param *kp) +{ + int dbcn, d, ret; + ret = kstrtoint(val, 0, &d); + if (ret < 0) + return ret; + dbcn = csr_read32(LOONGARCH_CSR_MWPC) & 0x3f; + if (d > dbcn) + return -EINVAL; + boot_cpu_data.watch_dreg_count = dbcn - d; + *(int *)kp->arg = d; + return 0; +} + +int param_set_icount(const char *val, const struct kernel_param *kp) +{ + int ibcn, d, ret; + ret = kstrtoint(val, 0, &d); + if (ret < 0) + return ret; + ibcn = csr_read32(LOONGARCH_CSR_FWPC) & 0x3f; + if (d > ibcn) + return -EINVAL; + boot_cpu_data.watch_ireg_count = ibcn - d; + *(int *)kp->arg = d; + return 0; +} + +const struct kernel_param_ops param_ops_dcount = { + .set = param_set_dcount, + .get = param_get_int, +}; + +const struct kernel_param_ops param_ops_icount = { + .set = param_set_icount, + .get = param_get_int, +}; + +module_param_cb(kgdb_watch_dcount, ¶m_ops_dcount, &kgdb_watch_dcount, 0644); +module_param_cb(kgdb_watch_icount, ¶m_ops_icount, &kgdb_watch_icount, 0644); + +static struct hard_trap_info { + unsigned char tt; /* Trap type code for LoongArch */ + unsigned char signo; /* Signal that we map this trap into */ +} hard_trap_info[] = { + { 1, SIGBUS }, + { 2, SIGBUS }, + { 3, SIGBUS }, + { 4, SIGBUS }, + { 5, SIGBUS }, + { 6, SIGBUS }, + { 7, SIGBUS }, + { 8, SIGBUS }, + { 9, SIGBUS }, + { 10, SIGBUS }, + { 12, SIGTRAP }, /* break */ + { 13, SIGBUS }, + { 14, SIGBUS }, + { 15, SIGFPE }, + { 16, SIGFPE }, + { 17, SIGFPE }, + { 18, SIGFPE }, + { 0, 0} /* Must be last */ +}; + +struct dbg_reg_def_t dbg_reg_def[DBG_ALL_REG_NUM] = { + { "r0", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[0]) }, + { "r1", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[1]) }, + { "r2", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[2]) }, + { "r3", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[3]) }, + { "r4", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[4]) }, + { "r5", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[5]) }, + { "r6", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[6]) }, + { "r7", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[7]) }, + { "r8", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[8]) }, + { "r9", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[9]) }, + { "r10", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[10]) }, + { "r11", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[11]) }, + { "r12", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[12]) }, + { "r13", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[13]) }, + { "r14", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[14]) }, + { "r15", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[15]) }, + { "r16", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[16]) }, + { "r17", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[17]) }, + { "r18", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[18]) }, + { "r19", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[19]) }, + { "r20", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[20]) }, + { "r21", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[21]) }, + { "r22", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[22]) }, + { "r23", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[23]) }, + { "r24", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[24]) }, + { "r25", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[25]) }, + { "r26", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[26]) }, + { "r27", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[27]) }, + { "r28", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[28]) }, + { "r29", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[29]) }, + { "r30", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[30]) }, + { "r31", GDB_SIZEOF_REG, offsetof(struct pt_regs, regs[31]) }, + { "orig_a0", GDB_SIZEOF_REG, offsetof(struct pt_regs, orig_a0) }, + { "pc", GDB_SIZEOF_REG, offsetof(struct pt_regs, csr_era) }, + { "badv", GDB_SIZEOF_REG, offsetof(struct pt_regs, csr_badvaddr) }, + { "f0", GDB_SIZEOF_REG, 0 }, + { "f1", GDB_SIZEOF_REG, 1 }, + { "f2", GDB_SIZEOF_REG, 2 }, + { "f3", GDB_SIZEOF_REG, 3 }, + { "f4", GDB_SIZEOF_REG, 4 }, + { "f5", GDB_SIZEOF_REG, 5 }, + { "f6", GDB_SIZEOF_REG, 6 }, + { "f7", GDB_SIZEOF_REG, 7 }, + { "f8", GDB_SIZEOF_REG, 8 }, + { "f9", GDB_SIZEOF_REG, 9 }, + { "f10", GDB_SIZEOF_REG, 10 }, + { "f11", GDB_SIZEOF_REG, 11 }, + { "f12", GDB_SIZEOF_REG, 12 }, + { "f13", GDB_SIZEOF_REG, 13 }, + { "f14", GDB_SIZEOF_REG, 14 }, + { "f15", GDB_SIZEOF_REG, 15 }, + { "f16", GDB_SIZEOF_REG, 16 }, + { "f17", GDB_SIZEOF_REG, 17 }, + { "f18", GDB_SIZEOF_REG, 18 }, + { "f19", GDB_SIZEOF_REG, 19 }, + { "f20", GDB_SIZEOF_REG, 20 }, + { "f21", GDB_SIZEOF_REG, 21 }, + { "f22", GDB_SIZEOF_REG, 22 }, + { "f23", GDB_SIZEOF_REG, 23 }, + { "f24", GDB_SIZEOF_REG, 24 }, + { "f25", GDB_SIZEOF_REG, 25 }, + { "f26", GDB_SIZEOF_REG, 26 }, + { "f27", GDB_SIZEOF_REG, 27 }, + { "f28", GDB_SIZEOF_REG, 28 }, + { "f29", GDB_SIZEOF_REG, 29 }, + { "f30", GDB_SIZEOF_REG, 30 }, + { "f31", GDB_SIZEOF_REG, 31 }, + { "fcc0", 1, 0 }, + { "fcc1", 1, 1 }, + { "fcc2", 1, 2 }, + { "fcc3", 1, 3 }, + { "fcc4", 1, 4 }, + { "fcc5", 1, 5 }, + { "fcc6", 1, 6 }, + { "fcc7", 1, 7 }, + { "fcsr", GDB_SIZEOF_REG, 0 }, +}; + +int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) +{ + int fp_reg; + + if (regno < 0 || regno >= DBG_ALL_REG_NUM) + return -EINVAL; + + if (dbg_reg_def[regno].offset != -1 && regno < 33) { + memcpy((void *)regs + dbg_reg_def[regno].offset, mem, + dbg_reg_def[regno].size); + } else if (current && dbg_reg_def[regno].offset != -1 && regno < 78) { + /* FP registers 35 -> 76 */ + if (!(regs->csr_euen & CSR_EUEN_FPEN)) + return 0; + if (regno == 75) { + /* Process the fcsr/fsr (register 75) */ + memcpy((void *)¤t->thread.fpu.fcsr, mem, + dbg_reg_def[regno].size); + } else if (regno >= 67 && regno < 75) { + /* Process the fcc */ + fp_reg = dbg_reg_def[regno].offset; + memcpy((char *)¤t->thread.fpu.fcc + fp_reg, mem, + dbg_reg_def[regno].size); + } else { + fp_reg = dbg_reg_def[regno].offset; + memcpy((void *)¤t->thread.fpu.fpr[fp_reg], mem, + dbg_reg_def[regno].size); + } + + restore_fp(current); + } + + return 0; +} + +char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) +{ + int fp_reg; + + if (regno >= DBG_ALL_REG_NUM || regno < 0) + return NULL; + + if (dbg_reg_def[regno].offset != -1 && regno < 33) { + /* First 32 registers */ + memcpy(mem, (void *)regs + dbg_reg_def[regno].offset, + dbg_reg_def[regno].size); + } else if (current && dbg_reg_def[regno].offset != -1 && regno < 78) { + /* FP registers 35 -> 76 */ + if (!(regs->csr_euen & CSR_EUEN_FPEN)) + goto out; + save_fp(current); + if (regno == 75) { + /* Process the fcsr/fsr (register 75) */ + memcpy(mem, (void *)¤t->thread.fpu.fcsr, + dbg_reg_def[regno].size); + } else if (regno >= 67 && regno < 75) { + /* Process the fcc */ + fp_reg = dbg_reg_def[regno].offset; + memcpy(mem, (char *)¤t->thread.fpu.fcc + fp_reg, + dbg_reg_def[regno].size); + } else { + fp_reg = dbg_reg_def[regno].offset; + memcpy(mem, (void *)¤t->thread.fpu.fpr[fp_reg], + dbg_reg_def[regno].size); + } + } + +out: + return dbg_reg_def[regno].name; + +} + +void arch_kgdb_breakpoint(void) +{ + __asm__ __volatile__( + ".globl breakinst\n\t" + "nop\n" + "breakinst:\tbreak 0\n\t"); + + annotate_reachable(); +} + +static int compute_signal(int tt) +{ + struct hard_trap_info *ht; + + for (ht = hard_trap_info; ht->tt && ht->signo; ht++) + if (ht->tt == tt) + return ht->signo; + + return SIGTRAP; /* default for things we don't know about */ +} + +/* + * Similar to regs_to_gdb_regs() except that process is sleeping and so + * we may not be able to get all the info. + */ +void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) +{ + int reg; +#if (KGDB_GDB_REG_SIZE == 32) + u32 *ptr = (u32 *)gdb_regs; +#else + u64 *ptr = (u64 *)gdb_regs; +#endif + + *(ptr++) = 0; + *(ptr++) = p->thread.reg01; + *(ptr++) = (long)p; + *(ptr++) = p->thread.reg03; + for (reg = 4; reg < 23; reg++) + *(ptr++) = 0; + + /* S0 - S8 */ + *(ptr++) = p->thread.reg23; + *(ptr++) = p->thread.reg24; + *(ptr++) = p->thread.reg25; + *(ptr++) = p->thread.reg26; + *(ptr++) = p->thread.reg27; + *(ptr++) = p->thread.reg28; + *(ptr++) = p->thread.reg29; + *(ptr++) = p->thread.reg30; + *(ptr++) = p->thread.reg31; + *(ptr++) = 0; + + /* + * PC use return address (RA), i.e. the moment after return from __switch_to() + */ + *(ptr++) = p->thread.reg01; +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) +{ + regs->csr_era = pc; +} + +/* + * Calls linux_debug_hook before the kernel dies. If KGDB is enabled, + * then try to fall into the debugger + */ +static int kgdb_loongarch_notify(struct notifier_block *self, unsigned long cmd, + void *ptr) +{ + struct die_args *args = (struct die_args *)ptr; + struct pt_regs *regs = args->regs; + int trap = read_csr_excode(); + +#ifdef CONFIG_KPROBES + /* + * Return immediately if the kprobes fault notifier has set + * DIE_PAGE_FAULT. + */ + if (cmd == DIE_PAGE_FAULT) + return NOTIFY_DONE; +#endif /* CONFIG_KPROBES */ + + /* Userspace events, ignore. */ + if (user_mode(regs)) + return NOTIFY_DONE; + + if (atomic_read(&kgdb_active) != -1) + kgdb_nmicallback(smp_processor_id(), regs); + + if (kgdb_handle_exception(trap, compute_signal(trap), cmd, regs)) + return NOTIFY_DONE; + + if (atomic_read(&kgdb_setting_breakpoint)) + if ((regs->csr_era == (unsigned long)breakinst)) + regs->csr_era += 4; + + /* In SMP mode, __flush_cache_all does IPI */ + local_irq_enable(); + flush_cache_all(); + + return NOTIFY_STOP; +} + +#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP +int kgdb_ll_trap(int cmd, const char *str, + struct pt_regs *regs, long err, int trap, int sig) +{ + struct die_args args = { + .regs = regs, + .str = str, + .err = err, + .trapnr = trap, + .signr = sig, + + }; + + if (!kgdb_io_module_registered) + return NOTIFY_DONE; + + return kgdb_loongarch_notify(NULL, cmd, &args); +} +#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ + +static struct notifier_block kgdb_notifier = { + .notifier_call = kgdb_loongarch_notify, +}; + +/* + * Handle the 'c' command + */ +int kgdb_arch_handle_exception(int vector, int signo, int err_code, + char *remcom_in_buffer, char *remcom_out_buffer, + struct pt_regs *regs) +{ + char *ptr; + unsigned long address; + + regs->csr_prmd |= CSR_PRMD_PWE; + + switch (remcom_in_buffer[0]) { + case 'c': + /* handle the optional parameter */ + ptr = &remcom_in_buffer[1]; + if (kgdb_hex2long(&ptr, &address)) + regs->csr_era = address; + + return 0; + } + + return -1; +} + +static struct hw_breakpoint { + unsigned enabled; + unsigned long addr; + int len; + int type; + struct perf_event * __percpu *pev; +} dbreakinfo[NUM_WATCH_REGS], ibreakinfo[NUM_WATCH_REGS]; + +static int +kgdb_set_hw_break(unsigned long addr, int len, enum kgdb_bptype bptype) +{ + int i; + struct hw_breakpoint *breakinfo = (bptype == BP_HARDWARE_BREAKPOINT) ? + ibreakinfo : dbreakinfo; + int count = (bptype == BP_HARDWARE_BREAKPOINT) ? kgdb_watch_icount : + kgdb_watch_dcount; + + for (i = 0; i < count; i++) + if (!breakinfo[i].enabled) + break; + if (i == count) + return -1; + + breakinfo[i].type = bptype; + breakinfo[i].len = len; + breakinfo[i].addr = addr; + breakinfo[i].enabled |= 1; + + return 0; +} + + +static int +kgdb_remove_hw_break(unsigned long addr, int len, enum kgdb_bptype bptype) +{ + int i; + struct hw_breakpoint *breakinfo = (bptype == BP_HARDWARE_BREAKPOINT) ? + ibreakinfo : dbreakinfo; + int count = (bptype == BP_HARDWARE_BREAKPOINT) ? kgdb_watch_icount : + kgdb_watch_dcount; + + for (i = 0; i < count; i++) + if (breakinfo[i].addr == addr && breakinfo[i].enabled) + break; + if (i == count) + return -1; + + breakinfo[i].enabled &= ~1; + + return 0; +} + +static void kgdb_disable_hw_debug(struct pt_regs *regs) +{ + csr_xchg32(0, CSR_CRMD_WE, LOONGARCH_CSR_CRMD); + regs->csr_prmd &= ~CSR_PRMD_PWE; +} + +static void kgdb_remove_all_hw_break(void) +{ + int i, j, mask; + + for (mask = 0, i = 0, j = boot_cpu_data.watch_ireg_count; + i < kgdb_watch_icount; i++, j++) { + if (!(ibreakinfo[i].enabled & 2)) + continue; + ibreakinfo[i].enabled = 0; + watch_csrwr(0, LOONGARCH_CSR_IB0ADDR + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_IB0MASK + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_IB0ASID + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_IB0CTL + 8 * j); + mask |= 1 << j; + } + watch_csrwr(mask, LOONGARCH_CSR_FWPS); + + for (mask = 0, i = 0, j = boot_cpu_data.watch_dreg_count; i < kgdb_watch_dcount; + i++, j++) { + if (!(dbreakinfo[i].enabled & 2)) + continue; + dbreakinfo[i].enabled = 0; + watch_csrwr(0, LOONGARCH_CSR_DB0ADDR + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_DB0MASK + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_DB0ASID + 8 * j); + watch_csrwr(0, LOONGARCH_CSR_DB0CTL + 8 * j); + mask |= 1 << j; + } + watch_csrwr(mask, LOONGARCH_CSR_MWPS); + + csr_xchg32(0, CSR_CRMD_WE, LOONGARCH_CSR_CRMD); + + kgdb_watch_activated = 0; +} + +static void kgdb_correct_hw_break(void) +{ + int i, j, dbc, activated = 0; + + for (i = 0, j = boot_cpu_data.watch_ireg_count; i < kgdb_watch_icount; i++, j++) { + if ((ibreakinfo[i].enabled & 3) == 2) { + watch_csrwr(0, LOONGARCH_CSR_IB0CTL + 8*j); + ibreakinfo[i].enabled = 0; + continue; + } else if (!ibreakinfo[i].enabled) + continue; + ibreakinfo[i].enabled |= 2; + watch_csrwr(ibreakinfo[i].addr, LOONGARCH_CSR_IB0ADDR + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0MASK + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0ASID + 8*j); + watch_csrwr(0x1e, LOONGARCH_CSR_IB0CTL + 8*j); + watch_csrwr(0x10000, LOONGARCH_CSR_FWPS); + activated = 1; + } + + for (i = 0, j = boot_cpu_data.watch_dreg_count; i < kgdb_watch_dcount; i++, j++) { + if ((dbreakinfo[i].enabled & 3) == 2) { + watch_csrwr(0, LOONGARCH_CSR_DB0CTL + 8*j); + dbreakinfo[i].enabled = 0; + continue; + } else if (!dbreakinfo[i].enabled) + continue; + dbreakinfo[i].enabled |= 2; + dbc = 0x1e; + switch (dbreakinfo[i].len) { + case 8: + break; + case 4: + dbc |= (1<<10); + break; + case 2: + dbc |= (2<<10); + break; + case 1: + dbc |= (3<<10); + break; + default: + break; + } + + if (dbreakinfo[i].type == BP_WRITE_WATCHPOINT) { + dbc |= 1<<9; + } else if (BP_READ_WATCHPOINT) { + dbc |= 1<<8; + } else { + dbc |= 3<<8; + } + + watch_csrwr(dbreakinfo[i].addr, LOONGARCH_CSR_DB0ADDR + 8*j); + watch_csrwr(0, LOONGARCH_CSR_DB0MASK + 8*j); + watch_csrwr(0, LOONGARCH_CSR_DB0ASID + 8*j); + watch_csrwr(dbc, LOONGARCH_CSR_DB0CTL + 8*j); + activated = 1; + } + + csr_xchg32(activated ? CSR_CRMD_WE : 0, CSR_CRMD_WE, LOONGARCH_CSR_CRMD); + kgdb_watch_activated = activated; +} + +const struct kgdb_arch arch_kgdb_ops = { + .flags = KGDB_HW_BREAKPOINT, + .set_hw_breakpoint = kgdb_set_hw_break, + .remove_hw_breakpoint = kgdb_remove_hw_break, + .disable_hw_break = kgdb_disable_hw_debug, + .remove_all_hw_break = kgdb_remove_all_hw_break, + .correct_hw_break = kgdb_correct_hw_break, + .gdb_bpt_instr = { 0x00, 0x00, break_op >> 1, 0x00 }, +}; + +int kgdb_arch_init(void) +{ + int ibcn, dbcn; + + register_die_notifier(&kgdb_notifier); + dbcn = csr_read32(LOONGARCH_CSR_MWPC) & 0x3f; + ibcn = csr_read32(LOONGARCH_CSR_FWPC) & 0x3f; + boot_cpu_data.watch_dreg_count = dbcn - kgdb_watch_dcount; + boot_cpu_data.watch_ireg_count = ibcn - kgdb_watch_icount; + return 0; +} + +/* + * kgdb_arch_exit - Perform any architecture specific uninitalization. + * + * This function will handle the uninitalization of any architecture + * specific callbacks, for dynamic registration and unregistration. + */ +void kgdb_arch_exit(void) +{ + unregister_die_notifier(&kgdb_notifier); +} diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c new file mode 100644 index 0000000000000000000000000000000000000000..bb4ed5f5df69832e1f728bd0a67d65fbc3a47e14 --- /dev/null +++ b/arch/loongarch/kernel/kprobes.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel Probes (KProbes) + * + * arch/loongarch/kernel/kprobes.c + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +static const union loongarch_instruction breakpoint_insn = { + .reg0i15_format = { + .opcode = break_op, + .simmediate = BRK_KPROBE_BP, + } +}; + +static const union loongarch_instruction breakpoint2_insn = { + .reg0i15_format = { + .opcode = break_op, + .simmediate = BRK_KPROBE_SSTEPBP, + } +}; + +DEFINE_PER_CPU(struct kprobe *, current_kprobe); +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +#define SS_FAIL -1 +#define SS_NONE 0 +#define SS_BRANCH 1 +#define SS_PC 2 + +/* + * insn_has_ll_or_sc function checks whether instruction is ll or sc + * one; putting breakpoint on top of atomic ll/sc pair is bad idea; + * so we need to prevent it and refuse kprobes insertion for such + * instructions; cannot do much about breakpoint in the middle of + * ll/sc pair; it is upto user to avoid those places + */ +static int __kprobes insn_has_ll_or_sc(union loongarch_instruction insn) +{ + int ret = 0; + + switch (insn.reg2i14_format.opcode) { + case llw_op: + case lld_op: + case scw_op: + case scd_op: + ret = 1; + break; + } + return ret; +} + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + union loongarch_instruction insn; + int ret = 0; + + insn = p->addr[0]; + + if (insn_has_ll_or_sc(insn)) { + pr_notice("Kprobes for ll and sc instructions are not" + "supported\n"); + ret = -EINVAL; + goto out; + } + + if (insn.reg1i21_format.opcode == bceqz_op) { + pr_notice("Kprobes for bceqz and bcnez instructions are not" + "supported\n"); + ret = -EINVAL; + goto out; + } + + /* insn: must be on special executable page on loongarch. */ + p->ainsn.insn = get_insn_slot(); + if (!p->ainsn.insn) { + ret = -ENOMEM; + goto out; + } + + /* + * In the kprobe->ainsn.insn[] array we store the original + * instruction at index zero and a break trap instruction at + * index one. + */ + memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t)); + + p->ainsn.insn[1] = breakpoint2_insn; + p->opcode = *p->addr; + +out: + return ret; +} + +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + *p->addr = breakpoint_insn; + flush_insn_slot(p); +} + +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + *p->addr = p->opcode; + flush_insn_slot(p); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } +} + +static void save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; + kcb->prev_kprobe.old_SR = kcb->kprobe_old_SR; + kcb->prev_kprobe.saved_SR = kcb->kprobe_saved_SR; + kcb->prev_kprobe.saved_era = kcb->kprobe_saved_era; +} + +static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; + kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR; + kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR; + kcb->kprobe_saved_era = kcb->prev_kprobe.saved_era; +} + +static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, p); + kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->csr_prmd & CSR_PRMD_PIE); + kcb->kprobe_saved_era = regs->csr_era; +} + +static int prepare_singlestep(struct kprobe *p, struct pt_regs *regs) +{ + if (is_branch_insn(p->opcode)) { + if (!simu_branch(regs, p->opcode)) + return SS_BRANCH; + } else if (is_pc_insn(p->opcode)) { + if (!simu_pc(regs, p->opcode)) + return SS_PC; + } else { + regs->csr_era = (unsigned long)&p->ainsn.insn[0]; + return SS_NONE; + } + + pr_notice("Kprobes: Error in simulate insn\n"); + regs->csr_era = (unsigned long)&p->ainsn.insn[0]; + return SS_FAIL; +} + +static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + int ss; + + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p, regs, kcb); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + /* single step inline if the instruction is an break */ + if (p->ainsn.insn->word == breakpoint_insn.word) { + regs->csr_prmd &= ~CSR_PRMD_PIE; + regs->csr_prmd |= kcb->kprobe_saved_SR; + preempt_enable_no_resched(); + } else { + regs->csr_prmd &= ~CSR_PRMD_PIE; + ss = prepare_singlestep(p, regs); + if (ss == SS_NONE) { + kcb->kprobe_status = KPROBE_HIT_SS; + } else if (ss == SS_BRANCH || ss == SS_PC) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (p->post_handler) + p->post_handler(p, regs, 0); + reset_current_kprobe(); + preempt_enable_no_resched(); + } else { + if (p->fault_handler) + p->fault_handler(p, regs, 0); + reset_current_kprobe(); + preempt_enable_no_resched(); + } + } +} + +static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_err("Unrecoverable kprobe detected. \n"); + BUG(); + break; + default: + /* impossible cases */ + WARN_ON(1); + return 0; + } + + return 1; +} + +static int __kprobes kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *p; + kprobe_opcode_t *addr; + struct kprobe_ctlblk *kcb; + + addr = (kprobe_opcode_t *) regs->csr_era; + + /* + * We don't want to be preempted for the entire + * duration of kprobe processing + */ + preempt_disable(); + kcb = get_kprobe_ctlblk(); + + p = get_kprobe(addr); + if (p) { + if (kprobe_running()) { + if (reenter_kprobe(p, regs, kcb)) + return 1; + } else { + set_current_kprobe(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (p->pre_handler && p->pre_handler(p, regs)) { + /* handler has already set things up, so skip ss setup */ + reset_current_kprobe(); + preempt_enable_no_resched(); + return 1; + } else { + setup_singlestep(p, regs, kcb, 0); + return 1; + } + } + } else { + if (addr->word != breakpoint_insn.word) { + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + */ + preempt_enable_no_resched(); + return 1; + } + } + + preempt_enable_no_resched(); + + return 0; +} + +static inline int post_kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (!cur) + return 0; + + if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + cur->post_handler(cur, regs, 0); + } + + regs->csr_era = kcb->kprobe_saved_era + LOONGARCH_INSN_SIZE; + regs->csr_prmd |= kcb->kprobe_saved_SR; + + /* Restore back the original saved kprobes variables and continue. */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + goto out; + } + reset_current_kprobe(); +out: + preempt_enable_no_resched(); + + return 1; +} + +int kprobe_fault_handler(struct pt_regs *regs, int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) + return 1; + + if (kcb->kprobe_status & KPROBE_HIT_SS) { + regs->csr_era = kcb->kprobe_saved_era + LOONGARCH_INSN_SIZE; + regs->csr_prmd |= kcb->kprobe_old_SR; + + reset_current_kprobe(); + preempt_enable_no_resched(); + } + return 0; +} + +/* + * Wrapper routine for handling exceptions. + */ +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + + struct die_args *args = (struct die_args *)data; + int ret = NOTIFY_DONE; + + switch (val) { + case DIE_BREAK: + if (kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + case DIE_SSTEPBP: + if (post_kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + + case DIE_PAGE_FAULT: + /* kprobe_running() needs smp_processor_id() */ + preempt_disable(); + + if (kprobe_running() + && kprobe_fault_handler(args->regs, args->trapnr)) + ret = NOTIFY_STOP; + preempt_enable(); + break; + default: + break; + } + return ret; +} + +/* + * Function return probe trampoline: + * - init_kprobes() establishes a probepoint here + * - When the probed function returns, this probe causes the + * handlers to fire + */ +static void __used kretprobe_trampoline_holder(void) +{ + asm volatile( + /* Keep the assembler from reordering and placing JR here. */ + "nop\n\t" + ".global kretprobe_trampoline\n" + "kretprobe_trampoline:\n\t" + "nop\n\t" + : : : "memory"); +} + +void kretprobe_trampoline(void); + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *) regs->regs[1]; + ri->fp = NULL; + + /* Replace the return addr with trampoline addr */ + regs->regs[1] = (unsigned long)kretprobe_trampoline; +} + +/* + * Called when the probe at kretprobe trampoline is hit + */ +static int __kprobes trampoline_probe_handler(struct kprobe *p, + struct pt_regs *regs) +{ + instruction_pointer(regs) = __kretprobe_trampoline_handler(regs, + kretprobe_trampoline, NULL); + /* + * By returning a non-zero value, we are telling + * kprobe_handler() that we don't want the post_handler + * to run (and have re-enabled preemption) + */ + return 1; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + if (p->addr == (kprobe_opcode_t *)kretprobe_trampoline) + return 1; + + return 0; +} + +static struct kprobe trampoline_p = { + .addr = (kprobe_opcode_t *)kretprobe_trampoline, + .pre_handler = trampoline_probe_handler +}; + +int __init arch_init_kprobes(void) +{ + return register_kprobe(&trampoline_p); +} diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c new file mode 100644 index 0000000000000000000000000000000000000000..a370b61058f3a18fc0244b4b2cb57f164644b03c --- /dev/null +++ b/arch/loongarch/kernel/machine_kexec.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * machine_kexec.c for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* 0x100000 ~ 0x200000 is safe */ +#define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL) +#define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL) + +static unsigned long reboot_code_buffer; +static cpumask_t cpus_in_crash = CPU_MASK_NONE; + +#ifdef CONFIG_SMP +static void (*relocated_kexec_smp_wait)(void *); +atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); +#endif + +static unsigned long efi_boot; +static unsigned long cmdline_ptr; +static unsigned long systable_ptr; +static unsigned long start_addr; +static unsigned long first_ind_entry; +static unsigned long kdump_reloc_offset; + +static void kexec_image_info(const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("kexec kimage info:\n"); + pr_debug("\ttype: %d\n", kimage->type); + pr_debug("\tstart: %lx\n", kimage->start); + pr_debug("\thead: %lx\n", kimage->head); + pr_debug("\tnr_segments: %lu\n", kimage->nr_segments); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug("\t segment[%lu]: %016lx - %016lx", i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz); + pr_debug("\t\t0x%lx bytes, %lu pages\n", + (unsigned long)kimage->segment[i].memsz, + (unsigned long)kimage->segment[i].memsz / PAGE_SIZE); + } +} + +int machine_kexec_prepare(struct kimage *kimage) +{ + int i; + char *bootloader = "kexec"; + void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; + + kexec_image_info(kimage); + + kimage->arch.efi_boot = fw_arg0; + kimage->arch.systable_ptr = fw_arg2; + + /* Find the command line */ + for (i = 0; i < kimage->nr_segments; i++) { + if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { + if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) + kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; + break; + } + } + + if (!kimage->arch.cmdline_ptr) { + pr_err("Command line not included in the provided image\n"); + return -EINVAL; + } + + /* kexec/kdump need a safe page to save reboot_code_buffer */ + kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE); + + reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page); + memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); + +#ifdef CONFIG_SMP + /* All secondary cpus now may jump to kexec_smp_wait cycle */ + relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel); +#endif + + return 0; +} + +void machine_kexec_cleanup(struct kimage *kimage) +{ +} + +void kexec_reboot(void) +{ + do_kexec_t do_kexec = NULL; + + /* + * We know we were online, and there will be no incoming IPIs at + * this point. Mark online again before rebooting so that the crash + * analysis tool will see us correctly. + */ + set_cpu_online(smp_processor_id(), true); + + /* Ensure remote CPUs observe that we're online before rebooting. */ + smp_mb__after_atomic(); + + /* + * Make sure we get correct instructions written by the + * machine_kexec_prepare() CPU. + */ + __asm__ __volatile__ ("\tibar 0\n"::); + +#ifdef CONFIG_SMP + /* All secondary cpus go to kexec_smp_wait */ + if (smp_processor_id() > 0) { + relocated_kexec_smp_wait(NULL); + unreachable(); + } +#endif + + do_kexec = (void *)reboot_code_buffer; + do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry, kdump_reloc_offset); + + unreachable(); +} + + +#ifdef CONFIG_SMP +static void kexec_shutdown_secondary(void *regs) +{ + int cpu = smp_processor_id(); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} + +static void crash_shutdown_secondary(void *passed_regs) +{ + int cpu = smp_processor_id(); + struct pt_regs *regs = passed_regs; + + /* + * If we are passed registers, use those. Otherwise get the + * regs from the last interrupt, which should be correct, as + * we are in an interrupt. But if the regs are not there, + * pull them from the top of the stack. They are probably + * wrong, but we need something to keep from crashing again. + */ + if (!regs) + regs = get_irq_regs(); + if (!regs) + regs = task_pt_regs(current); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + if (!cpumask_test_cpu(cpu, &cpus_in_crash)) + crash_save_cpu(regs, cpu); + cpumask_set_cpu(cpu, &cpus_in_crash); + + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} + +void crash_smp_send_stop(void) +{ + unsigned int ncpus; + unsigned long timeout; + static int cpus_stopped; + + kdump_reloc_offset = crashk_res.start; + + /* + * This function can be called twice in panic path, but obviously + * we should execute this only once. + */ + if (cpus_stopped) + return; + + cpus_stopped = 1; + + /* Excluding the panic cpu */ + ncpus = num_online_cpus() - 1; + + smp_call_function(crash_shutdown_secondary, NULL, 0); + smp_wmb(); + + /* + * The crash CPU sends an IPI and wait for other CPUs to + * respond. Delay of at least 10 seconds. + */ + timeout = MSEC_PER_SEC * 10; + pr_emerg("Sending IPI to other cpus...\n"); + while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) { + mdelay(1); + cpu_relax(); + } +} +#endif /* defined(CONFIG_SMP) */ + +void machine_shutdown(void) +{ + int cpu; + + /* All CPUs go to reboot_code_buffer */ + for_each_possible_cpu(cpu) + if (!cpu_online(cpu)) + cpu_device_up(get_cpu_device(cpu)); + +#ifdef CONFIG_SMP + smp_call_function(kexec_shutdown_secondary, NULL, 0); +#endif +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + int crashing_cpu; + + local_irq_disable(); + + crashing_cpu = smp_processor_id(); + crash_save_cpu(regs, crashing_cpu); + +#ifdef CONFIG_SMP + crash_smp_send_stop(); +#endif + cpumask_set_cpu(crashing_cpu, &cpus_in_crash); + + pr_info("Starting crashdump kernel...\n"); +} + +void machine_kexec(struct kimage *image) +{ + unsigned long entry, *ptr; + struct kimage_arch *internal = &image->arch; + + efi_boot = internal->efi_boot; + cmdline_ptr = internal->cmdline_ptr; + systable_ptr = internal->systable_ptr; + + start_addr = (unsigned long)phys_to_virt(image->start); + + first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ? + (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0; + + /* + * The generic kexec code builds a page list with physical + * addresses. they are directly accessible through XKPRANGE + * hence the phys_to_virt() call. + */ + for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); + ptr = (entry & IND_INDIRECTION) ? + phys_to_virt(entry & PAGE_MASK) : ptr + 1) { + if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION || + *ptr & IND_DESTINATION) + *ptr = (unsigned long) phys_to_virt(*ptr); + } + + /* Mark offline before disabling local irq. */ + set_cpu_online(smp_processor_id(), false); + + /* We do not want to be bothered. */ + local_irq_disable(); + + pr_notice("EFI boot flag 0x%lx\n", efi_boot); + pr_notice("Command line at 0x%lx\n", cmdline_ptr); + pr_notice("System table at 0x%lx\n", systable_ptr); + pr_notice("Kdump relocation offset 0x%lx\n", kdump_reloc_offset); + pr_notice("We will call new kernel at 0x%lx\n", start_addr); + pr_notice("Bye ...\n"); + + /* Make reboot code buffer available to the boot CPU. */ + flush_cache_all(); + +#ifdef CONFIG_SMP + atomic_set(&kexec_ready_to_reboot, 1); +#endif + + kexec_reboot(); +} diff --git a/arch/loongarch/kernel/mcount.S b/arch/loongarch/kernel/mcount.S new file mode 100644 index 0000000000000000000000000000000000000000..bbdbe9f91637b96fed1ec7ce27c45adaddbc480c --- /dev/null +++ b/arch/loongarch/kernel/mcount.S @@ -0,0 +1,94 @@ +/* + * LoongArch specific _mcount support + * + * Author: Huacai Chen + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + + .text + +#define MCOUNT_STACK_SIZE (2 * SZREG) +#define MCOUNT_S0_OFFSET (0) +#define MCOUNT_RA_OFFSET (SZREG) + + .macro MCOUNT_SAVE_REGS + PTR_ADDI sp, sp, -MCOUNT_STACK_SIZE + PTR_S s0, sp, MCOUNT_S0_OFFSET + PTR_S ra, sp, MCOUNT_RA_OFFSET + move s0, a0 + .endm + + .macro MCOUNT_RESTORE_REGS + move a0, s0 + PTR_L ra, sp, MCOUNT_RA_OFFSET + PTR_L s0, sp, MCOUNT_S0_OFFSET + PTR_ADDI sp, sp, MCOUNT_STACK_SIZE + .endm + + +SYM_FUNC_START(_mcount) + la t1, ftrace_stub + la t2, ftrace_trace_function /* Prepare t2 for (1) */ + PTR_L t2, t2, 0 + beq t1, t2, fgraph_trace + + MCOUNT_SAVE_REGS + + move a0, ra /* arg0: self return address */ + move a1, s0 /* arg1: parent's return address */ + jirl ra, t2, 0 /* (1) call *ftrace_trace_function */ + + MCOUNT_RESTORE_REGS + +fgraph_trace: +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + la t1, ftrace_stub + la t3, ftrace_graph_return + PTR_L t3, t3, 0 + bne t1, t3, ftrace_graph_caller + la t1, ftrace_graph_entry_stub + la t3, ftrace_graph_entry + PTR_L t3, t3, 0 + bne t1, t3, ftrace_graph_caller +#endif + + .globl ftrace_stub +ftrace_stub: + jirl zero, ra, 0 +SYM_FUNC_END(_mcount) +EXPORT_SYMBOL(_mcount) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +SYM_FUNC_START(ftrace_graph_caller) + MCOUNT_SAVE_REGS + + PTR_ADDI a0, ra, -4 /* arg0: Callsite self return addr */ + PTR_ADDI a1, sp, MCOUNT_STACK_SIZE /* arg1: Callsite sp */ + move a2, s0 /* arg2: Callsite parent ra */ + bl prepare_ftrace_return + + MCOUNT_RESTORE_REGS + jirl zero, ra, 0 +SYM_FUNC_END(ftrace_graph_caller) + +SYM_FUNC_START(return_to_handler) + PTR_ADDI sp, sp, -2 * SZREG + PTR_S a0, sp, 0 + PTR_S a1, sp, SZREG + + bl ftrace_return_to_handler + + /* restore the real parent address: a0 -> ra */ + move ra, a0 + + PTR_L a0, sp, 0 + PTR_L a1, sp, SZREG + PTR_ADDI sp, sp, 2 * SZREG + jirl zero, ra, 0 +SYM_FUNC_END(return_to_handler) +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S new file mode 100644 index 0000000000000000000000000000000000000000..5571c7327b900bbcbb3b848ea8e3ef4ee2599582 --- /dev/null +++ b/arch/loongarch/kernel/mcount_dyn.S @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Jinyang He + * Author: Huacai Chen + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include + + .text +/* + * Due to -fpatchable-function-entry=2: the compiler inserted 2 NOPs before the + * regular C function prologue. When PC arrived here, the last 2 instructions + * as follows, + * move t0, ra + * bl callsite (for modules, callsite is a tramplione) + * + * modules tramplione as follows, + * addu16i.d t1, zero, callsite[31:16] + * lu32i.d t1, callsite[51:32] + * lu52i.d t1, t1, callsite[63:52] + * jirl zero, t1, callsite[15:0] >> 2 + * + * See arch/loongarch/kernel/ftrace_dyn.c for details. Here, pay attention to + * that the T series regs are available and safe because each C functions + * follows the LoongArch psABI well. + */ + + .macro ftrace_regs_entry allregs=0 + PTR_ADDI sp, sp, -PT_SIZE + /* Save trace function ra at PT_ERA */ + PTR_S ra, sp, PT_ERA + /* Save parent ra at PT_R1(RA) */ + PTR_S t0, sp, PT_R1 + PTR_S a0, sp, PT_R4 + PTR_S a1, sp, PT_R5 + PTR_S a2, sp, PT_R6 + PTR_S a3, sp, PT_R7 + PTR_S a4, sp, PT_R8 + PTR_S a5, sp, PT_R9 + PTR_S a6, sp, PT_R10 + PTR_S a7, sp, PT_R11 + PTR_S fp, sp, PT_R22 + + .if \allregs + PTR_S t0, sp, PT_R12 + PTR_S t1, sp, PT_R13 + PTR_S t2, sp, PT_R14 + PTR_S t3, sp, PT_R15 + PTR_S t4, sp, PT_R16 + PTR_S t5, sp, PT_R17 + PTR_S t6, sp, PT_R18 + PTR_S t7, sp, PT_R19 + PTR_S t8, sp, PT_R20 + PTR_S s0, sp, PT_R23 + PTR_S s1, sp, PT_R24 + PTR_S s2, sp, PT_R25 + PTR_S s3, sp, PT_R26 + PTR_S s4, sp, PT_R27 + PTR_S s5, sp, PT_R28 + PTR_S s6, sp, PT_R29 + PTR_S s7, sp, PT_R30 + PTR_S s8, sp, PT_R31 + PTR_S tp, sp, PT_R2 + /* Clear it for later use as a flag sometimes. */ + PTR_S zero, sp, PT_R0 + PTR_S $r21, sp, PT_R21 + .endif + + PTR_ADDI t8, sp, PT_SIZE + PTR_S t8, sp, PT_R3 + + UNWIND_HINT_REGS + .endm + +SYM_CODE_START(ftrace_caller) + UNWIND_HINT sp_reg=ORC_REG_SP type=ORC_TYPE_CALL + ftrace_regs_entry allregs=0 + b ftrace_common +SYM_CODE_END(ftrace_caller) + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +SYM_CODE_START(ftrace_regs_caller) + UNWIND_HINT sp_reg=ORC_REG_SP type=ORC_TYPE_CALL + ftrace_regs_entry allregs=1 + b ftrace_common +SYM_CODE_END(ftrace_regs_caller) +#endif + +SYM_CODE_START(ftrace_common) + UNWIND_HINT_REGS + PTR_ADDI a0, ra, -8 /* arg0: ip */ + move a1, t0 /* arg1: parent_ip */ + la.pcrel t1, function_trace_op + PTR_L a2, t1, 0 /* arg2: op */ + move a3, sp /* arg3: regs */ + .globl ftrace_call +ftrace_call: + bl ftrace_stub +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .globl ftrace_graph_call +ftrace_graph_call: + nop /* b ftrace_graph_caller */ +#endif +/* + * As we didn't use S series regs in this assmembly code and all calls + * are C function which will save S series regs by themselves, there is + * no need to restore S series regs. The T series is available and safe + * at the callsite, so there is no need to restore the T series regs. + */ +ftrace_common_return: + PTR_L a0, sp, PT_R4 + PTR_L a1, sp, PT_R5 + PTR_L a2, sp, PT_R6 + PTR_L a3, sp, PT_R7 + PTR_L a4, sp, PT_R8 + PTR_L a5, sp, PT_R9 + PTR_L a6, sp, PT_R10 + PTR_L a7, sp, PT_R11 + PTR_L fp, sp, PT_R22 + PTR_L ra, sp, PT_R1 + PTR_L t0, sp, PT_ERA + PTR_ADDI sp, sp, PT_SIZE + UNWIND_HINT sp_reg=ORC_REG_SP type=ORC_TYPE_CALL + jirl zero, t0, 0 +SYM_CODE_END(ftrace_common) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +SYM_CODE_START(ftrace_graph_caller) + UNWIND_HINT_REGS + PTR_L a0, sp, PT_ERA + PTR_ADDI a0, a0, -8 /* arg0: self_addr */ + PTR_ADDI a1, sp, PT_R1 /* arg1: parent */ + bl prepare_ftrace_return + b ftrace_common_return +SYM_CODE_END(ftrace_graph_caller) + +SYM_CODE_START(return_to_handler) + UNWIND_HINT sp_reg=ORC_REG_SP type=ORC_TYPE_CALL + /* save return value regs */ + PTR_ADDI sp, sp, -2 * SZREG + PTR_S a0, sp, 0 + PTR_S a1, sp, SZREG + + move a0, zero /* Has no check FP now. */ + bl ftrace_return_to_handler + move ra, a0 /* parent ra */ + + /* restore return value regs */ + PTR_L a0, sp, 0 + PTR_L a1, sp, SZREG + PTR_ADDI sp, sp, 2 * SZREG + + jirl zero, ra, 0 +SYM_CODE_END(return_to_handler) +#endif + +SYM_FUNC_START(ftrace_stub) + jirl zero, ra, 0 +SYM_FUNC_END(ftrace_stub) diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c new file mode 100644 index 0000000000000000000000000000000000000000..e8e5bfed99826bd970f3964fed2e8aae9c593176 --- /dev/null +++ b/arch/loongarch/kernel/module-sections.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include + +Elf_Addr module_emit_got_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val) +{ + struct mod_section *got_sec = &mod->arch.got; + int i = got_sec->num_entries; + struct got_entry *got = get_got_entry(val, sechdrs, got_sec); + + if (got) + return (Elf_Addr)got; + + /* There is no GOT entry for val yet, create a new one. */ + got = (struct got_entry *)sechdrs[got_sec->shndx].sh_addr; + got[i] = emit_got_entry(val); + + got_sec->num_entries++; + if (got_sec->num_entries > got_sec->max_entries) { + /* + * This may happen when the module contains a GOT_HI20 without + * a paired GOT_LO12. Such a module is broken, reject it. + */ + pr_err("%s: module contains bad GOT relocation\n", mod->name); + return 0; + } + + return (Elf_Addr)&got[i]; +} + +Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val) +{ + int nr; + struct mod_section *plt_sec = &mod->arch.plt; + struct mod_section *plt_idx_sec = &mod->arch.plt_idx; + struct plt_entry *plt = get_plt_entry(val, sechdrs, plt_sec, plt_idx_sec); + struct plt_idx_entry *plt_idx; + + if (plt) + return (Elf_Addr)plt; + + nr = plt_sec->num_entries; + + /* There is no duplicate entry, create a new one */ + plt = (struct plt_entry *)sechdrs[plt_sec->shndx].sh_addr; + plt[nr] = emit_plt_entry(val); + plt_idx = (struct plt_idx_entry *)sechdrs[plt_idx_sec->shndx].sh_addr; + plt_idx[nr] = emit_plt_idx_entry(val); + + plt_sec->num_entries++; + plt_idx_sec->num_entries++; + BUG_ON(plt_sec->num_entries > plt_sec->max_entries); + + return (Elf_Addr)&plt[nr]; +} + +static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y) +{ + return x->r_info == y->r_info && x->r_addend == y->r_addend; +} + +static bool duplicate_rela(const Elf_Rela *rela, int idx) +{ + int i; + + for (i = 0; i < idx; i++) { + if (is_rela_equal(&rela[i], &rela[idx])) + return true; + } + + return false; +} + +static void count_max_entries(Elf_Rela *relas, int num, + unsigned int *plts, unsigned int *gots) +{ + unsigned int i, type; + + for (i = 0; i < num; i++) { + type = ELF_R_TYPE(relas[i].r_info); + switch (type) { + case R_LARCH_SOP_PUSH_PLT_PCREL: + case R_LARCH_B26: + if (!duplicate_rela(relas, i)) + (*plts)++; + break; + case R_LARCH_GOT_PC_HI20: + if (!duplicate_rela(relas, i)) + (*gots)++; + break; + default: + break; /* Do nothing. */ + } + } +} + +int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, + char *secstrings, struct module *mod) +{ + unsigned int i, num_plts = 0, num_gots = 0; + Elf_Shdr *got_sec, *plt_sec, *plt_idx_sec, *tramp = NULL; + + /* + * Find the empty .plt sections. + */ + for (i = 0; i < ehdr->e_shnum; i++) { + if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) + mod->arch.got.shndx = i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt")) + mod->arch.plt.shndx = i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx")) + mod->arch.plt_idx.shndx = i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".ftrace_trampoline")) + tramp = sechdrs + i; + } + + if (!mod->arch.got.shndx) { + pr_err("%s: module GOT section(s) missing\n", mod->name); + return -ENOEXEC; + } + if (!mod->arch.plt.shndx) { + pr_err("%s: module PLT section(s) missing\n", mod->name); + return -ENOEXEC; + } + if (!mod->arch.plt_idx.shndx) { + pr_err("%s: module PLT.IDX section(s) missing\n", mod->name); + return -ENOEXEC; + } + + /* Calculate the maxinum number of entries */ + for (i = 0; i < ehdr->e_shnum; i++) { + int num_rela = sechdrs[i].sh_size / sizeof(Elf_Rela); + Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset; + Elf_Shdr *dst_sec = sechdrs + sechdrs[i].sh_info; + + if (sechdrs[i].sh_type != SHT_RELA) + continue; + + /* ignore relocations that operate on non-exec sections */ + if (!(dst_sec->sh_flags & SHF_EXECINSTR)) + continue; + + count_max_entries(relas, num_rela, &num_plts, &num_gots); + } + + got_sec = sechdrs + mod->arch.got.shndx; + got_sec->sh_type = SHT_NOBITS; + got_sec->sh_flags = SHF_ALLOC; + got_sec->sh_addralign = L1_CACHE_BYTES; + got_sec->sh_size = (num_gots + 1) * sizeof(struct got_entry); + mod->arch.got.num_entries = 0; + mod->arch.got.max_entries = num_gots; + + plt_sec = sechdrs + mod->arch.plt.shndx; + plt_sec->sh_type = SHT_NOBITS; + plt_sec->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + plt_sec->sh_addralign = L1_CACHE_BYTES; + plt_sec->sh_size = (num_plts + 1) * sizeof(struct plt_entry); + mod->arch.plt.num_entries = 0; + mod->arch.plt.max_entries = num_plts; + + plt_idx_sec = sechdrs + mod->arch.plt_idx.shndx; + plt_idx_sec->sh_type = SHT_NOBITS; + plt_idx_sec->sh_flags = SHF_ALLOC; + plt_idx_sec->sh_addralign = L1_CACHE_BYTES; + plt_idx_sec->sh_size = (num_plts + 1) * sizeof(struct plt_idx_entry); + mod->arch.plt_idx.num_entries = 0; + mod->arch.plt_idx.max_entries = num_plts; + + if (tramp) { + tramp->sh_type = SHT_NOBITS; + tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + tramp->sh_addralign = __alignof__(struct plt_entry); + tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry); + } + + return 0; +} diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c new file mode 100644 index 0000000000000000000000000000000000000000..5ae87872c37e9e951ab64371bc1f096e571b7454 --- /dev/null +++ b/arch/loongarch/kernel/module.c @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +* Author: Huacai Chen +*/ + +#define pr_fmt(fmt) "kmod: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline bool signed_imm_check(long val, unsigned int bit) +{ + return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1)); +} + +static inline bool unsigned_imm_check(unsigned long val, unsigned int bit) +{ + return val < (1UL << bit); +} + +static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) +{ + if (RELA_STACK_DEPTH <= *rela_stack_top) + return -ENOEXEC; + + rela_stack[(*rela_stack_top)++] = stack_value; + pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); + + return 0; +} + +static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) +{ + if (*rela_stack_top == 0) + return -ENOEXEC; + + *stack_value = rela_stack[--(*rela_stack_top)]; + pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); + + return 0; +} + +static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + return 0; +} + +static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); + return -EINVAL; +} + +static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + *location = v; + return 0; +} + +static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + *(Elf_Addr *)location = v; + return 0; +} + +static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); +} + +static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + return rela_stack_push(v, rela_stack, rela_stack_top); +} + +static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + int err = 0; + s64 opr1; + + err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); + if (err) + return err; + err = rela_stack_push(opr1, rela_stack, rela_stack_top); + if (err) + return err; + err = rela_stack_push(opr1, rela_stack, rela_stack_top); + if (err) + return err; + + return 0; +} + +static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, + Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + ptrdiff_t offset = (void *)v - (void *)location; + + if (offset >= SZ_128M) + v = module_emit_plt_entry(mod, sechdrs, v); + + if (offset < -SZ_128M) + v = module_emit_plt_entry(mod, sechdrs, v); + + return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); +} + +static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + int err = 0; + s64 opr1, opr2, opr3; + + if (type == R_LARCH_SOP_IF_ELSE) { + err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); + if (err) + return err; + } + + err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); + if (err) + return err; + err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); + if (err) + return err; + + switch (type) { + case R_LARCH_SOP_AND: + err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); + break; + case R_LARCH_SOP_ADD: + err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); + break; + case R_LARCH_SOP_SUB: + err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); + break; + case R_LARCH_SOP_SL: + err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); + break; + case R_LARCH_SOP_SR: + err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); + break; + case R_LARCH_SOP_IF_ELSE: + err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); + break; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + + return err; +} + +static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + int err = 0; + s64 opr1; + union loongarch_instruction *insn = (union loongarch_instruction *)location; + + err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); + if (err) + return err; + + switch (type) { + case R_LARCH_SOP_POP_32_U_10_12: + if (!unsigned_imm_check(opr1, 12)) + goto overflow; + + /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ + insn->reg2ui12_format.simmediate = opr1 & 0xfff; + return 0; + case R_LARCH_SOP_POP_32_S_10_12: + if (!signed_imm_check(opr1, 12)) + goto overflow; + + insn->reg2i12_format.simmediate = opr1 & 0xfff; + return 0; + case R_LARCH_SOP_POP_32_S_10_16: + if (!signed_imm_check(opr1, 16)) + goto overflow; + + insn->reg2i16_format.simmediate = opr1 & 0xffff; + return 0; + case R_LARCH_SOP_POP_32_S_10_16_S2: + if (opr1 % 4) + goto unaligned; + + if (!signed_imm_check(opr1, 18)) + goto overflow; + + insn->reg2i16_format.simmediate = (opr1 >> 2) & 0xffff; + return 0; + case R_LARCH_SOP_POP_32_S_5_20: + if (!signed_imm_check(opr1, 20)) + goto overflow; + + insn->reg1i20_format.simmediate = (opr1) & 0xfffff; + return 0; + case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: + if (opr1 % 4) + goto unaligned; + + if (!signed_imm_check(opr1, 23)) + goto overflow; + + opr1 >>= 2; + insn->reg1i21_format.simmediate_l = opr1 & 0xffff; + insn->reg1i21_format.simmediate_h = (opr1 >> 16) & 0x1f; + return 0; + case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: + if (opr1 % 4) + goto unaligned; + + if (!signed_imm_check(opr1, 28)) + goto overflow; + + opr1 >>= 2; + insn->reg0i26_format.simmediate_l = opr1 & 0xffff; + insn->reg0i26_format.simmediate_h = (opr1 >> 16) & 0x3ff; + return 0; + case R_LARCH_SOP_POP_32_U: + if (!unsigned_imm_check(opr1, 32)) + goto overflow; + + /* (*(uint32_t *) PC) = opr */ + *location = (u32)opr1; + return 0; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + +overflow: + pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", + mod->name, opr1, __func__, type); + return -ENOEXEC; + +unaligned: + pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", + mod->name, opr1, __func__, type); + return -ENOEXEC; +} + +static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + switch (type) { + case R_LARCH_ADD32: + *(s32 *)location += v; + return 0; + case R_LARCH_ADD64: + *(s64 *)location += v; + return 0; + case R_LARCH_SUB32: + *(s32 *)location -= v; + return 0; + case R_LARCH_SUB64: + *(s64 *)location -= v; + return 0; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } +} + +static int apply_r_larch_b26(struct module *mod, + Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + ptrdiff_t offset = (void *)v - (void *)location; + union loongarch_instruction *insn = (union loongarch_instruction *)location; + + if (offset >= SZ_128M) + v = module_emit_plt_entry(mod, sechdrs, v); + + if (offset < -SZ_128M) + v = module_emit_plt_entry(mod, sechdrs, v); + + offset = (void *)v - (void *)location; + + if (offset & 3) { + pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", + mod->name, (long long)offset, type); + return -ENOEXEC; + } + + if (!signed_imm_check(offset, 28)) { + pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", + mod->name, (long long)offset, type); + return -ENOEXEC; + } + + offset >>= 2; + insn->reg0i26_format.simmediate_l = offset & 0xffff; + insn->reg0i26_format.simmediate_h = (offset >> 16) & 0x3ff; + + return 0; +} + +static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + union loongarch_instruction *insn = (union loongarch_instruction *)location; + /* Use s32 for a sign-extension deliberately. */ + s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - + (void *)((Elf_Addr)location & ~0xfff); + Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; + ptrdiff_t offset_rem = (void *)v - (void *)anchor; + + switch (type) { + case R_LARCH_PCALA_LO12: + insn->reg2i12_format.simmediate = v & 0xfff; + break; + case R_LARCH_PCALA_HI20: + v = offset_hi20 >> 12; + insn->reg1i20_format.simmediate = v & 0xfffff; + break; + case R_LARCH_PCALA64_LO20: + v = offset_rem >> 32; + insn->reg1i20_format.simmediate = v & 0xfffff; + break; + case R_LARCH_PCALA64_HI12: + v = offset_rem >> 52; + insn->reg2i12_format.simmediate = v & 0xfff; + break; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + + return 0; +} + +static int apply_r_larch_got_pc(struct module *mod, + Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); + + if (!got) + return -EINVAL; + + switch (type) { + case R_LARCH_GOT_PC_LO12: + type = R_LARCH_PCALA_LO12; + break; + case R_LARCH_GOT_PC_HI20: + type = R_LARCH_PCALA_HI20; + break; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + + return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); +} + +static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + ptrdiff_t offset = (void *)v - (void *)location; + + *(u32 *)location = offset; + return 0; +} + +static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + ptrdiff_t offset = (void *)v - (void *)location; + + *(u64 *)location = offset; + return 0; +} + +/* + * reloc_handlers_rela() - Apply a particular relocation to a module + * @mod: the module to apply the reloc to + * @location: the address at which the reloc is to be applied + * @v: the value of the reloc, with addend for RELA-style + * @rela_stack: the stack used for store relocation info, LOCAL to THIS module + * @rela_stac_top: where the stack operation(pop/push) applies to + * + * Return: 0 upon success, else -ERRNO + */ +typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type); + +/* The handlers for known reloc types */ +static reloc_rela_handler reloc_rela_handlers[] = { + [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error, + + [R_LARCH_NONE] = apply_r_larch_none, + [R_LARCH_32] = apply_r_larch_32, + [R_LARCH_64] = apply_r_larch_64, + [R_LARCH_MARK_LA] = apply_r_larch_none, + [R_LARCH_MARK_PCREL] = apply_r_larch_none, + [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, + [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, + [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, + [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, + [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, + [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, + [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, + [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel, + [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel, +}; + +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *mod) +{ + int i, err; + unsigned int type; + s64 rela_stack[RELA_STACK_DEPTH]; + size_t rela_stack_top = 0; + reloc_rela_handler handler; + void *location; + Elf_Addr v; + Elf_Sym *sym; + Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; + + pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, + sechdrs[relsec].sh_info); + + rela_stack_top = 0; + for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { + /* This is where to make the change */ + location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; + /* This is the symbol it is referring to */ + sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); + if (IS_ERR_VALUE(sym->st_value)) { + /* Ignore unresolved weak symbol */ + if (ELF_ST_BIND(sym->st_info) == STB_WEAK) + continue; + pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); + return -ENOENT; + } + + type = ELF_R_TYPE(rel[i].r_info); + + if (type < ARRAY_SIZE(reloc_rela_handlers)) + handler = reloc_rela_handlers[type]; + else + handler = NULL; + + if (!handler) { + pr_err("%s: Unknown relocation type %u\n", mod->name, type); + return -EINVAL; + } + + pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", + (int)ELF_R_TYPE(rel[i].r_info), + sym->st_value, rel[i].r_addend, (u64)location); + + v = sym->st_value + rel[i].r_addend; + switch (type) { + case R_LARCH_B26: + err = apply_r_larch_b26(mod, sechdrs, location, + v, rela_stack, &rela_stack_top, type); + break; + case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: + err = apply_r_larch_got_pc(mod, sechdrs, location, + v, rela_stack, &rela_stack_top, type); + break; + case R_LARCH_SOP_PUSH_PLT_PCREL: + err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, + v, rela_stack, &rela_stack_top, type); + break; + default: + err = handler(mod, location, v, rela_stack, &rela_stack_top, type); + } + if (err) + return err; + } + + return 0; +} + +void *module_alloc(unsigned long size) +{ + return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, + GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); +} + +static int module_init_ftrace_plt(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, struct module *mod) +{ +#ifdef CONFIG_DYNAMIC_FTRACE + struct plt_entry *ftrace_plts; + + ftrace_plts = (void *)sechdrs->sh_addr; + + ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); + + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) + ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); + + mod->arch.ftrace_trampolines = ftrace_plts; +#endif + return 0; +} + +int module_finalize(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, struct module *mod) +{ + char *secstrings; + const Elf_Shdr *s, *orc = NULL, *orc_ip = NULL, *alt = NULL, *ftrace = NULL; + + secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { + if (!strcmp(".altinstructions", secstrings + s->sh_name)) + alt = s; + if (!strcmp(".orc_unwind", secstrings + s->sh_name)) + orc = s; + if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) + orc_ip = s; + if (!strcmp(".ftrace_trampoline", secstrings + s->sh_name)) + ftrace = s; + } + + if (alt) + apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); + + if (orc && orc_ip) + unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); + + return module_init_ftrace_plt(hdr, ftrace, mod); +} diff --git a/arch/loongarch/kernel/paravirt-spinlocks.c b/arch/loongarch/kernel/paravirt-spinlocks.c new file mode 100644 index 0000000000000000000000000000000000000000..557119ae242bcb70466182014a5f9ee27d19b05c --- /dev/null +++ b/arch/loongarch/kernel/paravirt-spinlocks.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Split spinlock implementation out into its own file, so it can be + * compiled in a FTRACE-compatible way. + */ +#include +#include + +#include + +__visible void paravirt_wait_nop(u8 *ptr, u8 val) +{ +} + +__visible void paravirt_kick_nop(int cpu) +{ +} + +__visible void __native_queued_spin_unlock(struct qspinlock *lock) +{ + native_queued_spin_unlock(lock); +} + +bool pv_is_native_spin_unlock(void) +{ + return pv_lock_ops.queued_spin_unlock == __native_queued_spin_unlock; +} + +__visible bool __native_vcpu_is_preempted(long cpu) +{ + return false; +} + +bool pv_is_native_vcpu_is_preempted(void) +{ + return pv_lock_ops.vcpu_is_preempted == __native_vcpu_is_preempted; +} + +struct pv_lock_ops pv_lock_ops = { +#ifdef CONFIG_SMP + .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, + .queued_spin_unlock = __native_queued_spin_unlock, + .wait = paravirt_wait_nop, + .kick = paravirt_kick_nop, + .vcpu_is_preempted = __native_vcpu_is_preempted, +#endif /* SMP */ +}; +EXPORT_SYMBOL(pv_lock_ops); diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c new file mode 100644 index 0000000000000000000000000000000000000000..012c665fb29f40cac0f76108aa975f1cba9e4356 --- /dev/null +++ b/arch/loongarch/kernel/paravirt.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Loongson Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * LoongArch paravirtualization support. + * + * Author: wangjianxing + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; + +static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); +static int has_steal_clock; +static unsigned int pv_feature_disabled; + +static u64 dummy_steal_clock(int cpu) +{ + return 0; +} + +static u64 pv_steal_clock(int cpu) +{ + u64 steal; + struct kvm_steal_time *src; + int version; + + src = &per_cpu(steal_time, cpu); + do { + + version = src->version; + virt_rmb(); + steal = src->steal; + virt_rmb(); + + } while ((version & 1) || (version != src->version)); + return steal; +} + +struct pv_time_ops pv_time_ops = { + .steal_clock = dummy_steal_clock, +}; +EXPORT_SYMBOL_GPL(pv_time_ops); + +phys_addr_t slow_virt_to_phys(const void *vaddr) +{ + unsigned long addr = (unsigned long) vaddr; + unsigned long page_mask = 0; + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + phys_addr_t phys_addr; + + /* See arch/loongarch/la64/numa.c:setup_per_cpu_areas */ + if (nr_node_ids < 8) + return virt_to_phys((void *)vaddr); + + if (pgd_none(*pgd)) + goto out; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + goto out; + + pud = pud_offset(p4d, addr); + + if (pud_none(*pud) || pud_bad(*pud)) + goto out; + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + goto out; +#ifdef CONFIG_HUGETLB_PAGE + if (pmd_huge(*pmd)) { + pte = (pte_t *)pmd; + page_mask = PMD_MASK; + goto out; + } +#endif + pte = pte_offset_kernel(pmd, addr); + page_mask = PAGE_MASK; +out: + phys_addr = (pte_pfn(*pte) << PAGE_SHIFT) | (addr & ~page_mask); + return phys_addr; +} + +static void pv_register_steal_time(void) +{ + int cpu = smp_processor_id(); + struct kvm_steal_time *st; + + if (!has_steal_clock) + return; + + st = &per_cpu(steal_time, cpu); + pv_notify_host(KVM_FEATURE_STEAL_TIME, slow_virt_to_phys(st)); + + pr_info("pv stealtime: cpu %d, st:0x%llx phys:0x%llx\n", + cpu, (unsigned long long)st, (unsigned long long) slow_virt_to_phys(st)); +} + +#ifdef CONFIG_SMP +static void pv_disable_steal_time(void) +{ + if (has_steal_clock) + pv_notify_host(KVM_FEATURE_STEAL_TIME, 0); +} + +static int pv_cpu_online(unsigned int cpu) +{ + unsigned long flags; + local_irq_save(flags); + pv_register_steal_time(); + local_irq_restore(flags); + return 0; +} + +static int pv_cpu_down_prepare(unsigned int cpu) +{ + unsigned long flags; + local_irq_save(flags); + pv_disable_steal_time(); + local_irq_restore(flags); + return 0; +} +#endif + +static void pv_cpu_reboot(void *unused) +{ + pv_disable_steal_time(); +} + +static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, + void *unused) +{ + on_each_cpu(pv_cpu_reboot, NULL, 1); + return NOTIFY_DONE; +} + +static struct notifier_block pv_reboot_nb = { + .notifier_call = pv_reboot_notify, +}; + +int __init pv_time_init(void) +{ + if (!cpu_has_hypervisor) + return 0; + if (!kvm_para_available()) + return 0; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_STEAL_TIME)) && + pv_feature_support(KVM_FEATURE_STEAL_TIME)) { + + register_reboot_notifier(&pv_reboot_nb); + + has_steal_clock = 1; + pv_time_ops.steal_clock = pv_steal_clock; + pv_register_steal_time(); + static_key_slow_inc(¶virt_steal_enabled); + static_key_slow_inc(¶virt_steal_rq_enabled); + +#ifdef CONFIG_SMP + if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "loongarch/pv:online", + pv_cpu_online, pv_cpu_down_prepare) < 0) + pr_err("failed to install cpu hotplug callbacks\n"); +#endif + } + return 0; +} + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +static void kvm_wait(u8 *ptr, u8 val) +{ + if (READ_ONCE(*ptr) != val) + return; + + __arch_cpu_idle(); +} + +__visible bool __kvm_vcpu_is_preempted(long cpu) +{ + struct kvm_steal_time *src = &per_cpu(steal_time, cpu); + + return !!(src->preempted & KVM_VCPU_PREEMPTED); +} + +/* Kick a cpu. Used to wake up a halted vcpu */ +static void kvm_kick_cpu(int cpu) +{ + kvm_hypercall1(KVM_HC_KICK_CPU, cpu); +} + +/* + * Setup pv_lock_ops for guest kernel. + */ +void __init kvm_spinlock_init(void) +{ + if (!cpu_has_hypervisor) + return; + if (!kvm_para_available()) + return; + + /* Don't use the pvqspinlock code if there is only 1 vCPU. */ + if (num_possible_cpus() == 1) + return; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_PARAVIRT_SPINLOCK)) && + pv_feature_support(KVM_FEATURE_PARAVIRT_SPINLOCK)) { + + __pv_init_lock_hash(); + pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_lock_ops.queued_spin_unlock = __pv_queued_spin_unlock; + pv_lock_ops.wait = kvm_wait; + pv_lock_ops.kick = kvm_kick_cpu; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_STEAL_TIME)) && + pv_feature_support(KVM_FEATURE_STEAL_TIME)) { + pv_lock_ops.vcpu_is_preempted = __kvm_vcpu_is_preempted; + } + + pr_info("pv spinlock: supported\n"); + } +} +#endif + +static void pv_send_ipi(const struct cpumask *mask, unsigned int action) +{ + unsigned long flags; + unsigned int cpu, i, min = 0, max = 0; + u64 ipi_bitmap = 0; + long ret; + + if (cpumask_empty(mask)) + return; + + local_irq_save(flags); + + for_each_cpu(i, mask) { + cpu = cpu_logical_map(i); + if (!ipi_bitmap) { + min = max = cpu; + } else if (cpu < min && (max - cpu) < BITS_PER_LONG) { + ipi_bitmap <<= min - cpu; + min = cpu; + } else if (cpu > min && cpu < min + BITS_PER_LONG) { + max = cpu < max ? max : cpu; + } else { + ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action); + WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret); + min = max = cpu; + ipi_bitmap = 0; + } + __set_bit(cpu - min, (unsigned long *)&ipi_bitmap); + } + + if (ipi_bitmap) { + ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action); + WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret); + } + + local_irq_restore(flags); +} + +extern struct plat_smp_ops *mp_ops; +int __init pv_ipi_init(void) +{ + if (!cpu_has_hypervisor) + return 0; + if (!IS_ENABLED(CONFIG_SMP)) + return 0; + if (!kvm_para_available()) + return 0; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_MULTI_IPI)) && + pv_feature_support(KVM_FEATURE_MULTI_IPI)) { + mp_ops->send_ipi_mask = pv_send_ipi; + } + return 0; +} + +static int __init set_pv_ipi(char *str) +{ + pv_feature_disabled |= (1 << KVM_FEATURE_MULTI_IPI); + return 0; +} +early_param("no_pvipi", set_pv_ipi); + +static int __init set_pv_time(char *str) +{ + pv_feature_disabled |= (1 << KVM_FEATURE_STEAL_TIME); + return 0; +} +early_param("no_pvtime", set_pv_time); + +static int __init set_pv_spinlock(char *str) +{ + pv_feature_disabled |= (1 << KVM_FEATURE_PARAVIRT_SPINLOCK); + return 0; +} +early_param("no_pv_spinlock", set_pv_spinlock); diff --git a/arch/loongarch/kernel/perf_event.c b/arch/loongarch/kernel/perf_event.c new file mode 100644 index 0000000000000000000000000000000000000000..49174d08c36ae59d555caa01778ccd804afd3e41 --- /dev/null +++ b/arch/loongarch/kernel/perf_event.c @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Linux performance counter support for LoongArch. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Huacai Chen + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include /* For perf_irq */ +#include + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static unsigned long +user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) +{ + unsigned long err; + unsigned long __user *user_frame_tail; + struct stack_frame buftail; + + user_frame_tail = (unsigned long __user *)(fp - sizeof(struct stack_frame)); + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(user_frame_tail, sizeof(buftail))) + return 0; + + pagefault_disable(); + err = __copy_from_user_inatomic(&buftail, user_frame_tail, sizeof(buftail)); + pagefault_enable(); + + if (err || (unsigned long)user_frame_tail >= buftail.fp) + return 0; + + perf_callchain_store(entry, buftail.ra); + + return buftail.fp; +} + +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long fp; + + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + /* We don't support guest os callchain now */ + return; + } + + perf_callchain_store(entry, regs->csr_era); + + fp = regs->regs[22]; + + while (entry->nr < entry->max_stack && fp && !((unsigned long)fp & 0xf)) + fp = user_backtrace(entry, fp); +} + +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + struct unwind_state state; + unsigned long addr; + + for (unwind_start(&state, current, regs); + !unwind_done(&state); unwind_next_frame(&state)) { + addr = unwind_get_return_address(&state); + if (!addr || perf_callchain_store(entry, addr)) + return; + } +} + +#define LOONGARCH_MAX_HWEVENTS 32 + +struct cpu_hw_events { + /* Array of events on this cpu. */ + struct perf_event *events[LOONGARCH_MAX_HWEVENTS]; + + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(LOONGARCH_MAX_HWEVENTS)]; + + /* + * Software copy of the control register for each performance counter. + */ + unsigned int saved_ctrl[LOONGARCH_MAX_HWEVENTS]; +}; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { + .saved_ctrl = {0}, +}; + +/* The description of LoongArch performance events. */ +struct loongarch_perf_event { + unsigned int event_id; +}; + +static struct loongarch_perf_event raw_event; +static DEFINE_MUTEX(raw_event_mutex); + +#define C(x) PERF_COUNT_HW_CACHE_##x +#define HW_OP_UNSUPPORTED 0xffffffff +#define CACHE_OP_UNSUPPORTED 0xffffffff + +#define PERF_MAP_ALL_UNSUPPORTED \ + [0 ... PERF_COUNT_HW_MAX - 1] = {HW_OP_UNSUPPORTED} + +#define PERF_CACHE_MAP_ALL_UNSUPPORTED \ +[0 ... C(MAX) - 1] = { \ + [0 ... C(OP_MAX) - 1] = { \ + [0 ... C(RESULT_MAX) - 1] = {CACHE_OP_UNSUPPORTED}, \ + }, \ +} + +struct loongarch_pmu { + u64 max_period; + u64 valid_count; + u64 overflow; + const char *name; + unsigned int num_counters; + u64 (*read_counter)(unsigned int idx); + void (*write_counter)(unsigned int idx, u64 val); + const struct loongarch_perf_event *(*map_raw_event)(u64 config); + const struct loongarch_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; + const struct loongarch_perf_event (*cache_event_map) + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; +}; + +static struct loongarch_pmu loongarch_pmu; + +#define M_PERFCTL_EVENT(event) (event & CSR_PERFCTRL_EVENT) + +#define M_PERFCTL_COUNT_EVENT_WHENEVER (CSR_PERFCTRL_PLV0 | \ + CSR_PERFCTRL_PLV1 | \ + CSR_PERFCTRL_PLV2 | \ + CSR_PERFCTRL_PLV3 | \ + CSR_PERFCTRL_IE) + +#define M_PERFCTL_CONFIG_MASK 0x1f0000 + +static void pause_local_counters(void); +static void resume_local_counters(void); + +static u64 loongarch_pmu_read_counter(unsigned int idx) +{ + u64 val = -1; + + switch (idx) { + case 0: + val = read_csr_perfcntr0(); + break; + case 1: + val = read_csr_perfcntr1(); + break; + case 2: + val = read_csr_perfcntr2(); + break; + case 3: + val = read_csr_perfcntr3(); + break; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } + + return val; +} + +static void loongarch_pmu_write_counter(unsigned int idx, u64 val) +{ + switch (idx) { + case 0: + write_csr_perfcntr0(val); + return; + case 1: + write_csr_perfcntr1(val); + return; + case 2: + write_csr_perfcntr2(val); + return; + case 3: + write_csr_perfcntr3(val); + return; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return; + } +} + +static unsigned int loongarch_pmu_read_control(unsigned int idx) +{ + unsigned int val = -1; + + switch (idx) { + case 0: + val = read_csr_perfctrl0(); + break; + case 1: + val = read_csr_perfctrl1(); + break; + case 2: + val = read_csr_perfctrl2(); + break; + case 3: + val = read_csr_perfctrl3(); + break; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } + + return val; +} + +static void loongarch_pmu_write_control(unsigned int idx, unsigned int val) +{ + switch (idx) { + case 0: + write_csr_perfctrl0(val); + return; + case 1: + write_csr_perfctrl1(val); + return; + case 2: + write_csr_perfctrl2(val); + return; + case 3: + write_csr_perfctrl3(val); + return; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return; + } +} + +static int loongarch_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc) +{ + int i; + + for (i = 0; i < loongarch_pmu.num_counters; i++) { + if (!test_and_set_bit(i, cpuc->used_mask)) + return i; + } + + return -EAGAIN; +} + +static void loongarch_pmu_enable_event(struct hw_perf_event *evt, int idx) +{ + unsigned int cpu; + struct perf_event *event = container_of(evt, struct perf_event, hw); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + /* Make sure interrupt enabled. */ + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | CSR_PERFCTRL_IE; + + cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id(); + + /* + * We do not actually let the counter run. Leave it until start(). + */ + pr_debug("Enabling perf counter for CPU%d\n", cpu); +} + +static void loongarch_pmu_disable_event(int idx) +{ + unsigned long flags; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + local_irq_save(flags); + cpuc->saved_ctrl[idx] = loongarch_pmu_read_control(idx) & + ~M_PERFCTL_COUNT_EVENT_WHENEVER; + loongarch_pmu_write_control(idx, cpuc->saved_ctrl[idx]); + local_irq_restore(flags); +} + +static int loongarch_pmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + int ret = 0; + u64 left = local64_read(&hwc->period_left); + u64 period = hwc->sample_period; + + if (unlikely((left + period) & (1ULL << 63))) { + /* left underflowed by more than period. */ + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } else if (unlikely((left + period) <= period)) { + /* left underflowed by less than period. */ + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (left > loongarch_pmu.max_period) { + left = loongarch_pmu.max_period; + local64_set(&hwc->period_left, left); + } + + local64_set(&hwc->prev_count, loongarch_pmu.overflow - left); + + loongarch_pmu.write_counter(idx, loongarch_pmu.overflow - left); + + perf_event_update_userpage(event); + + return ret; +} + +static void loongarch_pmu_event_update(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + u64 prev_raw_count, new_raw_count; + u64 delta; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = loongarch_pmu.read_counter(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = new_raw_count - prev_raw_count; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); +} + +static void loongarch_pmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + + hwc->state = 0; + + /* Set the period for the event. */ + loongarch_pmu_event_set_period(event, hwc, hwc->idx); + + /* Enable the event. */ + loongarch_pmu_enable_event(hwc, hwc->idx); +} + +static void loongarch_pmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!(hwc->state & PERF_HES_STOPPED)) { + /* We are working on a local event. */ + loongarch_pmu_disable_event(hwc->idx); + barrier(); + loongarch_pmu_event_update(event, hwc, hwc->idx); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; + } +} + +static int loongarch_pmu_add(struct perf_event *event, int flags) +{ + int idx, err = 0; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + + perf_pmu_disable(event->pmu); + + /* To look for a free counter for this event. */ + idx = loongarch_pmu_alloc_counter(cpuc, hwc); + if (idx < 0) { + err = idx; + goto out; + } + + /* + * If there is an event in the counter we are going to use then + * make sure it is disabled. + */ + event->hw.idx = idx; + loongarch_pmu_disable_event(idx); + cpuc->events[idx] = event; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + loongarch_pmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + perf_pmu_enable(event->pmu); + return err; +} + +static void loongarch_pmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + loongarch_pmu_stop(event, PERF_EF_UPDATE); + cpuc->events[idx] = NULL; + clear_bit(idx, cpuc->used_mask); + + perf_event_update_userpage(event); +} + +static void loongarch_pmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* Don't read disabled counters! */ + if (hwc->idx < 0) + return; + + loongarch_pmu_event_update(event, hwc, hwc->idx); +} + +static void loongarch_pmu_enable(struct pmu *pmu) +{ + resume_local_counters(); +} + +static void loongarch_pmu_disable(struct pmu *pmu) +{ + pause_local_counters(); +} + +static DEFINE_MUTEX(pmu_reserve_mutex); +static atomic_t active_events = ATOMIC_INIT(0); + +static void reset_counters(void *arg); +static int __hw_perf_event_init(struct perf_event *event); + +static void hw_perf_event_destroy(struct perf_event *event) +{ + if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) { + on_each_cpu(reset_counters, NULL, 1); + free_irq(get_pmc_irq(), &loongarch_pmu); + mutex_unlock(&pmu_reserve_mutex); + } +} + +static void handle_associated_event(struct cpu_hw_events *cpuc, int idx, + struct perf_sample_data *data, struct pt_regs *regs) +{ + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc = &event->hw; + + loongarch_pmu_event_update(event, hwc, idx); + data->period = event->hw.last_period; + if (!loongarch_pmu_event_set_period(event, hwc, idx)) + return; + + if (perf_event_overflow(event, data, regs)) + loongarch_pmu_disable_event(idx); +} + +static irqreturn_t pmu_handle_irq(int irq, void *dev) +{ + int n; + int handled = IRQ_NONE; + uint64_t counter; + struct pt_regs *regs; + struct perf_sample_data data; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + /* + * First we pause the local counters, so that when we are locked + * here, the counters are all paused. When it gets locked due to + * perf_disable(), the timer interrupt handler will be delayed. + * + * See also loongarch_pmu_start(). + */ + pause_local_counters(); + + regs = get_irq_regs(); + + perf_sample_data_init(&data, 0, 0); + + for (n = 0; n < loongarch_pmu.num_counters; n++) { + if (test_bit(n, cpuc->used_mask)) { + counter = loongarch_pmu.read_counter(n); + if (counter & loongarch_pmu.overflow) { + handle_associated_event(cpuc, n, &data, regs); + handled = IRQ_HANDLED; + } + } + } + + resume_local_counters(); + + /* + * Do all the work for the pending perf events. We can do this + * in here because the performance counter interrupt is a regular + * interrupt, not NMI. + */ + if (handled == IRQ_HANDLED) + irq_work_run(); + + return handled; +} + +static int loongarch_pmu_event_init(struct perf_event *event) +{ + int r, irq; + unsigned long flags; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + + default: + /* Init it to avoid false validate_group */ + event->hw.event_base = 0xffffffff; + return -ENOENT; + } + + if (event->cpu >= 0 && !cpu_online(event->cpu)) + return -ENODEV; + + irq = get_pmc_irq(); + flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_SUSPEND | IRQF_SHARED; + if (!atomic_inc_not_zero(&active_events)) { + mutex_lock(&pmu_reserve_mutex); + if (atomic_read(&active_events) == 0) { + r = request_irq(irq, pmu_handle_irq, flags, "Perf_PMU", &loongarch_pmu); + if (r < 0) { + mutex_unlock(&pmu_reserve_mutex); + pr_warn("PMU IRQ request failed\n"); + return -ENODEV; + } + } + atomic_inc(&active_events); + mutex_unlock(&pmu_reserve_mutex); + } + + return __hw_perf_event_init(event); +} + +static struct pmu pmu = { + .pmu_enable = loongarch_pmu_enable, + .pmu_disable = loongarch_pmu_disable, + .event_init = loongarch_pmu_event_init, + .add = loongarch_pmu_add, + .del = loongarch_pmu_del, + .start = loongarch_pmu_start, + .stop = loongarch_pmu_stop, + .read = loongarch_pmu_read, +}; + +static unsigned int loongarch_pmu_perf_event_encode(const struct loongarch_perf_event *pev) +{ + return M_PERFCTL_EVENT(pev->event_id); +} + +static const struct loongarch_perf_event *loongarch_pmu_map_general_event(int idx) +{ + const struct loongarch_perf_event *pev; + + pev = &(*loongarch_pmu.general_event_map)[idx]; + + if (pev->event_id == HW_OP_UNSUPPORTED) + return ERR_PTR(-ENOENT); + + return pev; +} + +static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct loongarch_perf_event *pev; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + pev = &((*loongarch_pmu.cache_event_map) + [cache_type] + [cache_op] + [cache_result]); + + if (pev->event_id == CACHE_OP_UNSUPPORTED) + return ERR_PTR(-ENOENT); + + return pev; +} + +static int validate_group(struct perf_event *event) +{ + struct cpu_hw_events fake_cpuc; + struct perf_event *sibling, *leader = event->group_leader; + + memset(&fake_cpuc, 0, sizeof(fake_cpuc)); + + if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) + return -EINVAL; + + for_each_sibling_event(sibling, leader) { + if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) + return -EINVAL; + } + + if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) + return -EINVAL; + + return 0; +} + +static void reset_counters(void *arg) +{ + int n; + int counters = loongarch_pmu.num_counters; + + for (n = 0; n < counters; n++) { + loongarch_pmu_write_control(n, 0); + loongarch_pmu.write_counter(n, 0); + } +} + +static const struct loongarch_perf_event loongson_new_event_map[PERF_COUNT_HW_MAX] = { + PERF_MAP_ALL_UNSUPPORTED, + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00 }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01 }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x08 }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x09 }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02 }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x03 }, +}; + +static const struct loongarch_perf_event loongson_new_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +PERF_CACHE_MAP_ALL_UNSUPPORTED, +[C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x8 }, + [C(RESULT_MISS)] = { 0x9 }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x8 }, + [C(RESULT_MISS)] = { 0x9 }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0xaa }, + [C(RESULT_MISS)] = { 0xa9 }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x6 }, + [C(RESULT_MISS)] = { 0x7 }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0xc }, + [C(RESULT_MISS)] = { 0xd }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0xc }, + [C(RESULT_MISS)] = { 0xd }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x3b }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x4 }, + [C(RESULT_MISS)] = { 0x3c }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x4 }, + [C(RESULT_MISS)] = { 0x3c }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x02 }, + [C(RESULT_MISS)] = { 0x03 }, + }, +}, +}; + +static int __hw_perf_event_init(struct perf_event *event) +{ + int err; + struct hw_perf_event *hwc = &event->hw; + struct perf_event_attr *attr = &event->attr; + const struct loongarch_perf_event *pev; + + /* Returning LoongArch event descriptor for generic perf event. */ + if (PERF_TYPE_HARDWARE == event->attr.type) { + if (event->attr.config >= PERF_COUNT_HW_MAX) + return -EINVAL; + pev = loongarch_pmu_map_general_event(event->attr.config); + } else if (PERF_TYPE_HW_CACHE == event->attr.type) { + pev = loongarch_pmu_map_cache_event(event->attr.config); + } else if (PERF_TYPE_RAW == event->attr.type) { + /* We are working on the global raw event. */ + mutex_lock(&raw_event_mutex); + pev = loongarch_pmu.map_raw_event(event->attr.config); + } else { + /* The event type is not (yet) supported. */ + return -EOPNOTSUPP; + } + + if (IS_ERR(pev)) { + if (PERF_TYPE_RAW == event->attr.type) + mutex_unlock(&raw_event_mutex); + return PTR_ERR(pev); + } + + /* + * We allow max flexibility on how each individual counter shared + * by the single CPU operates (the mode exclusion and the range). + */ + hwc->config_base = CSR_PERFCTRL_IE; + + hwc->event_base = loongarch_pmu_perf_event_encode(pev); + if (PERF_TYPE_RAW == event->attr.type) + mutex_unlock(&raw_event_mutex); + + if (!attr->exclude_user) { + hwc->config_base |= CSR_PERFCTRL_PLV3; + hwc->config_base |= CSR_PERFCTRL_PLV2; + } + if (!attr->exclude_kernel) { + hwc->config_base |= CSR_PERFCTRL_PLV0; + } + if (!attr->exclude_hv) { + hwc->config_base |= CSR_PERFCTRL_PLV1; + } + + hwc->config_base &= M_PERFCTL_CONFIG_MASK; + /* + * The event can belong to another cpu. We do not assign a local + * counter for it for now. + */ + hwc->idx = -1; + hwc->config = 0; + + if (!hwc->sample_period) { + hwc->sample_period = loongarch_pmu.max_period; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + err = 0; + if (event->group_leader != event) + err = validate_group(event); + + event->destroy = hw_perf_event_destroy; + + if (err) + event->destroy(event); + + return err; +} + +static void pause_local_counters(void) +{ + unsigned long flags; + int ctr = loongarch_pmu.num_counters; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + local_irq_save(flags); + do { + ctr--; + cpuc->saved_ctrl[ctr] = loongarch_pmu_read_control(ctr); + loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] & + ~M_PERFCTL_COUNT_EVENT_WHENEVER); + } while (ctr > 0); + local_irq_restore(flags); +} + +static void resume_local_counters(void) +{ + int ctr = loongarch_pmu.num_counters; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + do { + ctr--; + loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]); + } while (ctr > 0); +} + +static const struct loongarch_perf_event *loongarch_pmu_map_raw_event(u64 config) +{ + /* currently most cores have 10-bit event numbers */ + unsigned int raw_id = M_PERFCTL_EVENT(config); + unsigned int base_id = M_PERFCTL_EVENT(raw_id); + + switch (current_cpu_data.cputype) { + case CPU_LOONGSON64: + base_id = raw_id; + break; + default: + panic("Unkown cpu type '0x%x\n", current_cpu_data.cputype); + } + + raw_event.event_id = base_id; + + return &raw_event; +} + +static int __init init_hw_perf_events(void) +{ + int counters; + + if (!cpu_has_pmp) + return -ENODEV; + + pr_info("Performance counters: "); + counters = ((read_cpucfg(LOONGARCH_CPUCFG6) & CPUCFG6_PMNUM) >> 4) + 1; + + loongarch_pmu.map_raw_event = loongarch_pmu_map_raw_event; + + switch (current_cpu_data.cputype) { + case CPU_LOONGSON64: + loongarch_pmu.name = "loongarch/loongson64"; + loongarch_pmu.general_event_map = &loongson_new_event_map; + loongarch_pmu.cache_event_map = &loongson_new_cache_map; + break; + default: + pr_cont("Either hardware does not support performance " + "counters, or not yet implemented.\n"); + return -ENODEV; + } + + loongarch_pmu.num_counters = counters; + loongarch_pmu.max_period = (1ULL << 63) - 1; + loongarch_pmu.valid_count = (1ULL << 63) - 1; + loongarch_pmu.overflow = 1ULL << 63; + loongarch_pmu.read_counter = loongarch_pmu_read_counter; + loongarch_pmu.write_counter = loongarch_pmu_write_counter; + + on_each_cpu(reset_counters, NULL, 1); + + pr_cont("%s PMU enabled, %d %d-bit counters available to each CPU.\n", + loongarch_pmu.name, counters, 64); + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c new file mode 100644 index 0000000000000000000000000000000000000000..b13b1a4475b4d2d73968895131cfc904f00f4c7e --- /dev/null +++ b/arch/loongarch/kernel/perf_regs.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Some parts derived from x86 version of this file. + * + * Copyright (C) 2013 Cavium, Inc. + * Copyright (C) 2021 Loongson Technology Corporation Limited + */ + +#include + +#include + +#ifdef CONFIG_32BIT +u64 perf_reg_abi(struct task_struct *tsk) +{ + return PERF_SAMPLE_REGS_ABI_32; +} +#else /* Must be CONFIG_64BIT */ +u64 perf_reg_abi(struct task_struct *tsk) +{ + if (test_tsk_thread_flag(tsk, TIF_32BIT_REGS)) + return PERF_SAMPLE_REGS_ABI_32; + else + return PERF_SAMPLE_REGS_ABI_64; +} +#endif /* CONFIG_32BIT */ + +int perf_reg_validate(u64 mask) +{ + if (!mask) + return -EINVAL; + if (mask & ~((1ull << PERF_REG_LOONGARCH_MAX) - 1)) + return -EINVAL; + return 0; +} + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + if (WARN_ON_ONCE((u32)idx >= PERF_REG_LOONGARCH_MAX)) + return 0; + + if ((u32)idx == PERF_REG_LOONGARCH_PC) + return regs->csr_era; + + return regs->regs[idx]; +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c new file mode 100644 index 0000000000000000000000000000000000000000..9dcd2158a81560f0b0343d3860987736376216f1 --- /dev/null +++ b/arch/loongarch/kernel/proc.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * * No lock; only written during early bootup by CPU 0. + * */ +static RAW_NOTIFIER_HEAD(proc_cpuinfo_chain); + +int __ref register_proc_cpuinfo_notifier(struct notifier_block *nb) +{ + return raw_notifier_chain_register(&proc_cpuinfo_chain, nb); +} + +int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v) +{ + return raw_notifier_call_chain(&proc_cpuinfo_chain, val, v); +} + +static int show_cpuinfo(struct seq_file *m, void *v) +{ + unsigned long n = (unsigned long) v - 1; + unsigned int version = cpu_data[n].processor_id & 0xff; + unsigned int fp_version = cpu_data[n].fpu_vers; + struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; + +#ifdef CONFIG_SMP + if (!cpu_online(n)) + return 0; +#endif + + /* + * For the first processor also print the system type + */ + if (n == 0) + seq_printf(m, "system type\t\t: %s\n\n", get_system_type()); + + seq_printf(m, "processor\t\t: %ld\n", n); + seq_printf(m, "package\t\t\t: %d\n", cpu_data[n].package); + seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core); + seq_printf(m, "global_id\t\t: %d\n", cpu_data[n].global_id); + seq_printf(m, "CPU Family\t\t: %s\n", __cpu_family[n]); + seq_printf(m, "Model Name\t\t: %s\n", __cpu_full_name[n]); + seq_printf(m, "CPU Revision\t\t: 0x%02x\n", version); + seq_printf(m, "FPU Revision\t\t: 0x%02x\n", fp_version); + seq_printf(m, "CPU MHz\t\t\t: %llu.%02llu\n", + cpu_clock_freq / 1000000, (cpu_clock_freq / 10000) % 100); + seq_printf(m, "BogoMIPS\t\t: %llu.%02llu\n", + (lpj_fine * cpu_clock_freq / const_clock_freq) / (500000/HZ), + ((lpj_fine * cpu_clock_freq / const_clock_freq) / (5000/HZ)) % 100); + seq_printf(m, "TLB Entries\t\t: %d\n", cpu_data[n].tlbsize); + seq_printf(m, "Address Sizes\t\t: %d bits physical, %d bits virtual\n", + cpu_pabits + 1, cpu_vabits + 1); + + seq_printf(m, "ISA\t\t\t:"); + if (cpu_has_loongarch32) + seq_printf(m, " loongarch32"); + if (cpu_has_loongarch64) + seq_printf(m, " loongarch64"); + seq_printf(m, "\n"); + + seq_printf(m, "Features\t\t:"); + if (cpu_has_cpucfg) seq_printf(m, " cpucfg"); + if (cpu_has_lam) seq_printf(m, " lam"); + if (cpu_has_ual) seq_printf(m, " ual"); + if (cpu_has_fpu) seq_printf(m, " fpu"); + if (cpu_has_lsx) seq_printf(m, " lsx"); + if (cpu_has_lasx) seq_printf(m, " lasx"); + if (cpu_has_crc32) seq_printf(m, " crc32"); + if (cpu_has_complex) seq_printf(m, " complex"); + if (cpu_has_crypto) seq_printf(m, " crypto"); + if (cpu_has_ptw) seq_printf(m, " ptw"); + if (cpu_has_lvz) seq_printf(m, " lvz"); + if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86"); + if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm"); + if (cpu_has_lbt_mips) seq_printf(m, " lbt_mips"); + seq_printf(m, "\n"); + + seq_printf(m, "Hardware Watchpoint\t: %s", + cpu_has_watch ? "yes, " : "no\n"); + if (cpu_has_watch) { + seq_printf(m, "iwatch count: %d, dwatch count: %d\n", + cpu_data[n].watch_ireg_count, cpu_data[n].watch_dreg_count); + } + + proc_cpuinfo_notifier_args.m = m; + proc_cpuinfo_notifier_args.n = n; + + raw_notifier_call_chain(&proc_cpuinfo_chain, 0, + &proc_cpuinfo_notifier_args); + + seq_printf(m, "\n"); + + return 0; +} + +static void *c_start(struct seq_file *m, loff_t *pos) +{ + unsigned long i = *pos; + + return i < nr_cpu_ids ? (void *) (i + 1) : NULL; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + ++*pos; + return c_start(m, pos); +} + +static void c_stop(struct seq_file *m, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = show_cpuinfo, +}; diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c new file mode 100644 index 0000000000000000000000000000000000000000..878c0e09f98471fdcf751950e190533522755f7a --- /dev/null +++ b/arch/loongarch/kernel/process.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Idle related variables and functions + */ + +unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; +EXPORT_SYMBOL(boot_option_idle_override); + +asmlinkage void ret_from_fork(void); +asmlinkage void ret_from_kernel_thread(void); + +void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) +{ + unsigned long crmd; + unsigned long prmd; + unsigned long euen; + + /* New thread loses kernel privileges. */ + crmd = regs->csr_crmd & ~(PLV_MASK); + crmd |= PLV_USER; + regs->csr_crmd = crmd; + + prmd = regs->csr_prmd & ~(PLV_MASK); + prmd |= PLV_USER; + regs->csr_prmd = prmd; + + euen = regs->csr_euen & ~(CSR_EUEN_FPEN); + regs->csr_euen = euen; + lose_fpu(0); + lose_lbt(0); + current->thread.fpu.fcsr = boot_cpu_data.fpu_csr0; + + clear_thread_flag(TIF_LSX_CTX_LIVE); + clear_thread_flag(TIF_LASX_CTX_LIVE); + clear_thread_flag(TIF_LBT_CTX_LIVE); + clear_used_math(); + regs->csr_era = pc; + regs->regs[3] = sp; +} + +void exit_thread(struct task_struct *tsk) +{ +} + +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) +{ + /* + * Save any process state which is live in hardware registers to the + * parent context prior to duplication. This prevents the new child + * state becoming stale if the parent is preempted before copy_thread() + * gets a chance to save the parent's live hardware registers to the + * child context. + */ + preempt_disable(); + + if (is_fpu_owner()) { + if (is_lasx_enabled()) + save_lasx(current); + else if (is_lsx_enabled()) + save_lsx(current); + else + save_fp(current); + } + + preempt_enable(); + + if (used_math()) + memcpy(dst, src, sizeof(struct task_struct)); + else + memcpy(dst, src, offsetof(struct task_struct, thread.fpu.fpr)); + + return 0; +} + +/* + * Copy architecture-specific thread state + */ +int copy_thread(unsigned long clone_flags, unsigned long usp, + unsigned long kthread_arg, struct task_struct *p, unsigned long tls) +{ + unsigned long childksp; + struct pt_regs *childregs, *regs = current_pt_regs(); + + childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE; + + /* set up new TSS. */ + childregs = (struct pt_regs *) childksp - 1; + /* Put the stack after the struct pt_regs. */ + childksp = (unsigned long) childregs; + p->thread.sched_cfa = 0; + p->thread.csr_euen = 0; + p->thread.csr_crmd = csr_read32(LOONGARCH_CSR_CRMD); + p->thread.csr_prmd = csr_read32(LOONGARCH_CSR_PRMD); + p->thread.csr_ecfg = csr_read32(LOONGARCH_CSR_ECFG); + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + p->thread.reg23 = usp; /* fn */ + p->thread.reg24 = kthread_arg; + p->thread.reg03 = childksp; + p->thread.reg01 = (unsigned long) ret_from_kernel_thread; + p->thread.sched_ra = (unsigned long) ret_from_kernel_thread; + memset(childregs, 0, sizeof(struct pt_regs)); + childregs->csr_euen = p->thread.csr_euen; + childregs->csr_crmd = p->thread.csr_crmd; + childregs->csr_prmd = p->thread.csr_prmd; + childregs->csr_ecfg = p->thread.csr_ecfg; + goto out; + } + + /* user thread */ + *childregs = *regs; + childregs->regs[4] = 0; /* Child gets zero as return value */ + if (usp) + childregs->regs[3] = usp; + + p->thread.reg03 = (unsigned long) childregs; + p->thread.reg01 = (unsigned long) ret_from_fork; + p->thread.sched_ra = (unsigned long) ret_from_fork; + + /* + * New tasks lose permission to use the fpu. This accelerates context + * switching for most programs since they don't use the fpu. + */ + childregs->csr_euen = 0; + + if (clone_flags & CLONE_SETTLS) + childregs->regs[2] = tls; + +out: + clear_tsk_thread_flag(p, TIF_USEDFPU); + clear_tsk_thread_flag(p, TIF_USEDSIMD); + clear_tsk_thread_flag(p, TIF_USEDLBT); + clear_tsk_thread_flag(p, TIF_LSX_CTX_LIVE); + clear_tsk_thread_flag(p, TIF_LASX_CTX_LIVE); + clear_tsk_thread_flag(p, TIF_LBT_CTX_LIVE); + + return 0; +} + +#ifdef CONFIG_STACKPROTECTOR +#include +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + +bool in_task_stack(unsigned long stack, struct task_struct *task, + struct stack_info *info) +{ + unsigned long begin = (unsigned long)task_stack_page(task); + unsigned long end = begin + THREAD_SIZE; + + if (stack < begin || stack >= end) + return false; + + info->type = STACK_TYPE_TASK; + info->begin = begin; + info->end = end; + info->next_sp = 0; + + return true; +} + +bool in_irq_stack(unsigned long stack, struct stack_info *info) +{ + unsigned long nextsp; + unsigned long begin = (unsigned long)this_cpu_read(irq_stack); + unsigned long end = begin + IRQ_STACK_START; + + if (stack < begin || stack >= end) + return false; + + nextsp = *(unsigned long *)end; + if (nextsp & (SZREG - 1)) + return false; + + info->type = STACK_TYPE_IRQ; + info->begin = begin; + info->end = end; + info->next_sp = nextsp; + + return true; +} + +int get_stack_info(unsigned long stack, struct task_struct *task, + struct stack_info *info) +{ + task = task ? : current; + + if (!stack || stack & (SZREG - 1)) + goto unknown; + + if (in_task_stack(stack, task, info)) + return 0; + + if (task != current) + goto unknown; + + if (in_irq_stack(stack, info)) + return 0; + +unknown: + info->type = STACK_TYPE_UNKNOWN; + return -EINVAL; +} + +unsigned long get_wchan(struct task_struct *task) +{ + unsigned long stack_page; + unsigned long pc; + unsigned long stack_page_end __maybe_unused; + unsigned long frame __maybe_unused; + struct unwind_state state __maybe_unused; + + if (!task || task == current || task->state == TASK_RUNNING) + return 0; + + stack_page = (unsigned long)try_get_task_stack(task); + if (!stack_page) + return 0; + + pc = thread_saved_ra(task); + +#ifdef CONFIG_UNWINDER_GUESS + /* + * Use "guess" unwinder get wchan is not reliable. + * Just calculate a simpler wchan value here. + * Do nothing here. + */ + return 0; +#endif + + stack_page_end = stack_page + THREAD_SIZE; + frame = thread_saved_fp(task); + + if (frame >= stack_page_end || frame < stack_page) + goto out; + + memset(&state, 0, sizeof(state)); + state.task = task; + state.sp = frame; + state.stack_info.type = STACK_TYPE_TASK; + state.stack_info.begin = stack_page; + state.stack_info.end = stack_page_end; + state.stack_info.next_sp = 0; + state.pc = pc; +#ifdef CONFIG_UNWINDER_PROLOGUE + state.enable = true; + state.first = true; +#endif + + while (in_sched_functions(pc)) { + if (!unwind_next_frame(&state)) { + pc = 0; + goto out; + } + pc = unwind_get_return_address(&state); + } + +out: + + put_task_stack(task); + return pc; +} + +unsigned long stack_top(void) +{ + unsigned long top = TASK_SIZE & PAGE_MASK; + + /* Space for the VDSO & data page */ + top -= PAGE_ALIGN(current->thread.vdso->size); + top -= VVAR_SIZE; + + /* Space to randomize the VDSO base */ + if (current->flags & PF_RANDOMIZE) + top -= VDSO_RANDOMIZE_SIZE; + + return top; +} + +/* + * Don't forget that the stack pointer must be aligned on a 8 bytes + * boundary for 32-bits ABI and 16 bytes for 64-bits ABI. + */ +unsigned long arch_align_stack(unsigned long sp) +{ + if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) + sp -= get_random_int() & ~PAGE_MASK; + + return sp & STACK_ALIGN; +} + +static DEFINE_PER_CPU(call_single_data_t, backtrace_csd); +static struct cpumask backtrace_csd_busy; + +static void handle_backtrace(void *info) +{ + nmi_cpu_backtrace(get_irq_regs()); + cpumask_clear_cpu(smp_processor_id(), &backtrace_csd_busy); +} + +static void raise_backtrace(cpumask_t *mask) +{ + call_single_data_t *csd; + int cpu; + + for_each_cpu(cpu, mask) { + /* + * If we previously sent an IPI to the target CPU & it hasn't + * cleared its bit in the busy cpumask then it didn't handle + * our previous IPI & it's not safe for us to reuse the + * call_single_data_t. + */ + if (cpumask_test_and_set_cpu(cpu, &backtrace_csd_busy)) { + pr_warn("Unable to send backtrace IPI to CPU%u - perhaps it hung?\n", + cpu); + continue; + } + + csd = &per_cpu(backtrace_csd, cpu); + csd->func = handle_backtrace; + smp_call_function_single_async(cpu, csd); + } +} + +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +{ + nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace); +} + +#ifdef CONFIG_64BIT +void loongarch_dump_regs64(u64 *uregs, const struct pt_regs *regs) +{ + unsigned int i; + + for (i = LOONGARCH_EF_R1; i <= LOONGARCH_EF_R31; i++) { + uregs[i] = regs->regs[i - LOONGARCH_EF_R0]; + } + + uregs[LOONGARCH_EF_ORIG_A0] = regs->orig_a0; + uregs[LOONGARCH_EF_CSR_ERA] = regs->csr_era; + uregs[LOONGARCH_EF_CSR_BADV] = regs->csr_badvaddr; + uregs[LOONGARCH_EF_CSR_CRMD] = regs->csr_crmd; + uregs[LOONGARCH_EF_CSR_PRMD] = regs->csr_prmd; + uregs[LOONGARCH_EF_CSR_EUEN] = regs->csr_euen; + uregs[LOONGARCH_EF_CSR_ECFG] = regs->csr_ecfg; + uregs[LOONGARCH_EF_CSR_ESTAT] = regs->csr_estat; +} +#endif /* CONFIG_64BIT */ diff --git a/arch/loongarch/kernel/ptrace.c b/arch/loongarch/kernel/ptrace.c new file mode 100644 index 0000000000000000000000000000000000000000..7b87a38ffbdda500f24ef75592b0167ad651d3bc --- /dev/null +++ b/arch/loongarch/kernel/ptrace.c @@ -0,0 +1,710 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +* Author: Huacai Chen +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void init_fp_ctx(struct task_struct *target) +{ + /* The target already has context */ + if (tsk_used_math(target)) + return; + + /* Begin with data registers set to all 1s... */ + memset(&target->thread.fpu.fpr, ~0, sizeof(target->thread.fpu.fpr)); + set_stopped_child_used_math(target); +} + +/* + * Called by kernel/ptrace.c when detaching.. + * + * Make sure single step bits etc are not set. + */ +void ptrace_disable(struct task_struct *child) +{ + /* Don't load the watchpoint registers for the ex-child. */ + clear_tsk_thread_flag(child, TIF_LOAD_WATCH); + clear_tsk_thread_flag(child, TIF_SINGLESTEP); +} + +/* regset get/set implementations */ + +static int gpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + int r; + struct pt_regs *regs = task_pt_regs(target); + + r = membuf_write(&to, ®s->regs, sizeof(u64) * GPR_NUM); + r = membuf_write(&to, ®s->orig_a0, sizeof(u64)); + r = membuf_write(&to, ®s->csr_era, sizeof(u64)); + r = membuf_write(&to, ®s->csr_badvaddr, sizeof(u64)); + + return r; +} + +static int gpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int err; + int a0_start = sizeof(u64) * GPR_NUM; + int era_start = a0_start + sizeof(u64); + int badvaddr_start = era_start + sizeof(u64); + struct pt_regs *regs = task_pt_regs(target); + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->regs, + 0, a0_start); + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->orig_a0, + a0_start, a0_start + sizeof(u64)); + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->csr_era, + era_start, era_start + sizeof(u64)); + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->csr_badvaddr, + badvaddr_start, badvaddr_start + sizeof(u64)); + + return err; +} + + +/* + * Get the general floating-point registers. + */ +static int gfpr_get(struct task_struct *target, struct membuf *to) +{ + return membuf_write(to, &target->thread.fpu.fpr, + sizeof(elf_fpreg_t) * NUM_FPU_REGS); +} + +static int gfpr_get_simd(struct task_struct *target, struct membuf *to) +{ + int i, r; + u64 fpr_val; + + BUILD_BUG_ON(sizeof(fpr_val) != sizeof(elf_fpreg_t)); + for (i = 0; i < NUM_FPU_REGS; i++) { + fpr_val = get_fpr64(&target->thread.fpu.fpr[i], 0); + r = membuf_write(to, &fpr_val, sizeof(elf_fpreg_t)); + } + + return r; +} + +/* + * Choose the appropriate helper for general registers, and then copy + * the FCC and FCSR registers separately. + */ +static int fpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + int r; + + save_fpu_regs(target); + + if (sizeof(target->thread.fpu.fpr[0]) == sizeof(elf_fpreg_t)) + r = gfpr_get(target, &to); + else + r = gfpr_get_simd(target, &to); + + r = membuf_write(&to, &target->thread.fpu.fcc, sizeof(target->thread.fpu.fcc)); + r = membuf_write(&to, &target->thread.fpu.fcsr, sizeof(target->thread.fpu.fcsr)); + + return r; +} + +static int gfpr_set(struct task_struct *target, + unsigned int *pos, unsigned int *count, + const void **kbuf, const void __user **ubuf) +{ + return user_regset_copyin(pos, count, kbuf, ubuf, + &target->thread.fpu.fpr, + 0, NUM_FPU_REGS * sizeof(elf_fpreg_t)); +} + +static int gfpr_set_simd(struct task_struct *target, + unsigned int *pos, unsigned int *count, + const void **kbuf, const void __user **ubuf) +{ + int i, err; + u64 fpr_val; + + BUILD_BUG_ON(sizeof(fpr_val) != sizeof(elf_fpreg_t)); + for (i = 0; i < NUM_FPU_REGS && *count > 0; i++) { + err = user_regset_copyin(pos, count, kbuf, ubuf, + &fpr_val, i * sizeof(elf_fpreg_t), + (i + 1) * sizeof(elf_fpreg_t)); + if (err) + return err; + set_fpr64(&target->thread.fpu.fpr[i], 0, fpr_val); + } + + return 0; +} + +/* + * Choose the appropriate helper for general registers, and then copy + * the FCC register separately. + */ +static int fpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + const int fcc_start = NUM_FPU_REGS * sizeof(elf_fpreg_t); + const int fcsr_start = fcc_start + sizeof(u64); + int err; + + BUG_ON(count % sizeof(elf_fpreg_t)); + if (pos + count > sizeof(elf_fpregset_t)) + return -EIO; + + init_fp_ctx(target); + + if (sizeof(target->thread.fpu.fpr[0]) == sizeof(elf_fpreg_t)) + err = gfpr_set(target, &pos, &count, &kbuf, &ubuf); + else + err = gfpr_set_simd(target, &pos, &count, &kbuf, &ubuf); + if (err) + return err; + + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fcc, fcc_start, + fcc_start + sizeof(u64)); + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fcsr, fcsr_start, + fcsr_start + sizeof(u32)); + + return err; +} + +static int cfg_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + int i, r; + u32 cfg_val; + + i = 0; + while (to.left > 0) { + cfg_val = read_cpucfg(i++); + r = membuf_write(&to, &cfg_val, sizeof(u32)); + } + + return r; +} + +/* + * CFG registers are read-only. + */ +static int cfg_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + return 0; +} + +#ifdef CONFIG_CPU_HAS_LBT +static int lbt_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + int r; + + r = membuf_write(&to, &target->thread.lbt.scr0, sizeof(target->thread.lbt.scr0)); + r = membuf_write(&to, &target->thread.lbt.scr1, sizeof(target->thread.lbt.scr1)); + r = membuf_write(&to, &target->thread.lbt.scr2, sizeof(target->thread.lbt.scr2)); + r = membuf_write(&to, &target->thread.lbt.scr3, sizeof(target->thread.lbt.scr3)); + r = membuf_write(&to, &target->thread.fpu.ftop, sizeof(target->thread.fpu.ftop)); + + return r; +} + +static int lbt_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int err; + const int ftop_start = (regset->n - 1) * regset->size; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.lbt.scr0, 0, ftop_start); + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.ftop, ftop_start, + ftop_start + sizeof(u64)); + return err; +} +#endif + +#ifdef CONFIG_CPU_HAS_LSX + +static void copy_pad_fprs(struct task_struct *target, + const struct user_regset *regset, + struct membuf *to, unsigned int live_sz) +{ + int i, j; + unsigned long long fill = ~0ull; + unsigned int cp_sz, pad_sz; + + cp_sz = min(regset->size, live_sz); + pad_sz = regset->size - cp_sz; + WARN_ON(pad_sz % sizeof(fill)); + + for (i = 0; i < NUM_FPU_REGS; i++) { + membuf_write(to, &target->thread.fpu.fpr[i], cp_sz); + for (j = 0; j < (pad_sz / sizeof(fill)); j++) { + membuf_store(to, fill); + } + } +} + +static int simd_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + + save_fpu_regs(target); + + if (!tsk_used_math(target)) { + /* The task hasn't used FP or LSX, fill with 0xff */ + copy_pad_fprs(target, regset, &to, 0); + } else if (!test_tsk_thread_flag(target, TIF_LSX_CTX_LIVE)) { + /* Copy scalar FP context, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, 8); +#ifdef CONFIG_CPU_HAS_LASX + } else if (!test_tsk_thread_flag(target, TIF_LASX_CTX_LIVE)) { + /* Copy LSX 128 Bit context, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, 16); +#endif + } else if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + membuf_write(&to, &target->thread.fpu.fpr, wr_size); + } else { + /* Copy as much context as possible, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, sizeof(target->thread.fpu.fpr[0])); + } + + return 0; +} + +static int simd_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + unsigned int cp_sz; + int i, err, start; + + init_fp_ctx(target); + + if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr, + 0, wr_size); + } else { + /* Copy as much context as possible */ + cp_sz = min_t(unsigned int, regset->size, + sizeof(target->thread.fpu.fpr[0])); + + i = start = err = 0; + for (; i < NUM_FPU_REGS; i++, start += regset->size) { + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr[i], + start, start + cp_sz); + } + } + + return err; +} + +#endif /* CONFIG_CPU_HAS_LSX */ + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(n, r) {.name = #n, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(r0, regs[0]), + REG_OFFSET_NAME(r1, regs[1]), + REG_OFFSET_NAME(r2, regs[2]), + REG_OFFSET_NAME(r3, regs[3]), + REG_OFFSET_NAME(r4, regs[4]), + REG_OFFSET_NAME(r5, regs[5]), + REG_OFFSET_NAME(r6, regs[6]), + REG_OFFSET_NAME(r7, regs[7]), + REG_OFFSET_NAME(r8, regs[8]), + REG_OFFSET_NAME(r9, regs[9]), + REG_OFFSET_NAME(r10, regs[10]), + REG_OFFSET_NAME(r11, regs[11]), + REG_OFFSET_NAME(r12, regs[12]), + REG_OFFSET_NAME(r13, regs[13]), + REG_OFFSET_NAME(r14, regs[14]), + REG_OFFSET_NAME(r15, regs[15]), + REG_OFFSET_NAME(r16, regs[16]), + REG_OFFSET_NAME(r17, regs[17]), + REG_OFFSET_NAME(r18, regs[18]), + REG_OFFSET_NAME(r19, regs[19]), + REG_OFFSET_NAME(r20, regs[20]), + REG_OFFSET_NAME(r21, regs[21]), + REG_OFFSET_NAME(r22, regs[22]), + REG_OFFSET_NAME(r23, regs[23]), + REG_OFFSET_NAME(r24, regs[24]), + REG_OFFSET_NAME(r25, regs[25]), + REG_OFFSET_NAME(r26, regs[26]), + REG_OFFSET_NAME(r27, regs[27]), + REG_OFFSET_NAME(r28, regs[28]), + REG_OFFSET_NAME(r29, regs[29]), + REG_OFFSET_NAME(r30, regs[30]), + REG_OFFSET_NAME(r31, regs[31]), + REG_OFFSET_NAME(orig_a0, orig_a0), + REG_OFFSET_NAME(csr_era, csr_era), + REG_OFFSET_NAME(csr_badvaddr, csr_badvaddr), + REG_OFFSET_NAME(csr_crmd, csr_crmd), + REG_OFFSET_NAME(csr_prmd, csr_prmd), + REG_OFFSET_NAME(csr_euen, csr_euen), + REG_OFFSET_NAME(csr_ecfg, csr_ecfg), + REG_OFFSET_NAME(csr_estat, csr_estat), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +enum loongarch_regset { + REGSET_GPR, + REGSET_FPR, + REGSET_CPUCFG, +#ifdef CONFIG_CPU_HAS_LBT + REGSET_LBT, +#endif +#ifdef CONFIG_CPU_HAS_LSX + REGSET_LSX, +#endif +#ifdef CONFIG_CPU_HAS_LASX + REGSET_LASX, +#endif +}; + +static const struct user_regset loongarch64_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(elf_greg_t), + .align = sizeof(elf_greg_t), + .regset_get = gpr_get, + .set = gpr_set, + }, + [REGSET_FPR] = { + .core_note_type = NT_PRFPREG, + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .regset_get = fpr_get, + .set = fpr_set, + }, + [REGSET_CPUCFG] = { + .core_note_type = NT_LOONGARCH_CPUCFG, + .n = 64, + .size = sizeof(u32), + .align = sizeof(u32), + .regset_get = cfg_get, + .set = cfg_set, + }, +#ifdef CONFIG_CPU_HAS_LBT + [REGSET_LBT] = { + .core_note_type = NT_LOONGARCH_LBT, + .n = 6, + .size = sizeof(u64), + .align = sizeof(u64), + .regset_get = lbt_get, + .set = lbt_set, + }, +#endif +#ifdef CONFIG_CPU_HAS_LSX + [REGSET_LSX] = { + .core_note_type = NT_LOONGARCH_LSX, + .n = NUM_FPU_REGS, + .size = 16, + .align = 16, + .regset_get = simd_get, + .set = simd_set, + }, +#endif +#ifdef CONFIG_CPU_HAS_LASX + [REGSET_LASX] = { + .core_note_type = NT_LOONGARCH_LASX, + .n = NUM_FPU_REGS, + .size = 32, + .align = 32, + .regset_get = simd_get, + .set = simd_set, + }, +#endif +}; + +static const struct user_regset_view user_loongarch64_view = { + .name = "loongarch64", + .e_machine = ELF_ARCH, + .regsets = loongarch64_regsets, + .n = ARRAY_SIZE(loongarch64_regsets), +}; + + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &user_loongarch64_view; +} + +static inline int read_user(struct task_struct *target, unsigned long addr, + unsigned long __user *data) +{ + unsigned long tmp = 0; + + switch (addr) { + case 0 ... 31: + tmp = task_pt_regs(target)->regs[addr]; + break; + case ARG0: + tmp = task_pt_regs(target)->orig_a0; + break; + case PC: + tmp = task_pt_regs(target)->csr_era; + break; + case BADVADDR: + tmp = task_pt_regs(target)->csr_badvaddr; + break; + default: + return -EIO; + } + + return put_user(tmp, data); +} + +static inline int write_user(struct task_struct *target, unsigned long addr, + unsigned long data) +{ + switch (addr) { + case 0 ... 31: + task_pt_regs(target)->regs[addr] = data; + break; + case ARG0: + task_pt_regs(target)->orig_a0 = data; + break; + case PC: + task_pt_regs(target)->csr_era = data; + break; + case BADVADDR: + task_pt_regs(target)->csr_badvaddr = data; + break; + default: + return -EIO; + } + + return 0; +} + +static int ptrace_get_watch_regs(struct task_struct *child, + struct pt_watch_regs __user *addr) +{ + enum pt_watch_style style; + int i; + unsigned int cnt; + + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) + return -EIO; + if (!access_ok(addr, sizeof(struct pt_watch_regs))) + return -EIO; + +#ifdef CONFIG_32BIT + style = pt_watch_style_la32; +#define WATCH_STYLE la32 +#else + style = pt_watch_style_la64; +#define WATCH_STYLE la64 +#endif + + loongarch_update_watch_registers(child); + + /* Reserve the first instruction watchpoint if TIF_SINGLESTEP is set. */ + if (unlikely(test_thread_flag(TIF_SINGLESTEP))) + child->thread.watch.irwmask[boot_cpu_data.watch_dreg_count] = 0; + + __get_user(cnt, &addr->max_valid); + cnt = min(boot_cpu_data.watch_reg_use_cnt, cnt); + __put_user(cnt, &addr->num_valid); + __put_user(style, &addr->style); + for (i = 0; i < cnt; i++) { + __put_user(child->thread.watch.addr[i], &addr->WATCH_STYLE[i].addr); + __put_user(child->thread.watch.mask[i], &addr->WATCH_STYLE[i].mask); + __put_user(child->thread.watch.irw[i], &addr->WATCH_STYLE[i].irw); + __put_user(child->thread.watch.irwstat[i], &addr->WATCH_STYLE[i].irwstat); + __put_user(child->thread.watch.irwmask[i], &addr->WATCH_STYLE[i].irwmask); + } + + return 0; +} + +static int ptrace_set_watch_regs(struct task_struct *child, + struct pt_watch_regs __user *addr) +{ + int i; + unsigned int cnt; + int watch_active = 0; + unsigned long addrt[NUM_WATCH_REGS]; + unsigned long maskt[NUM_WATCH_REGS]; + unsigned char irwt[NUM_WATCH_REGS]; + + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) + return -EIO; + if (!access_ok(addr, sizeof(struct pt_watch_regs))) + return -EIO; + + __get_user(cnt, &addr->max_valid); + cnt = min(boot_cpu_data.watch_reg_use_cnt, cnt); + /* Check the values. */ + for (i = 0; i < cnt; i++) { + __get_user(addrt[i], &addr->WATCH_STYLE[i].addr); +#ifdef CONFIG_32BIT + if (addrt[i] & __UA_LIMIT) + return -EINVAL; +#else + if (test_tsk_thread_flag(child, TIF_32BIT_ADDR)) { + if (addrt[i] & 0xffffffff80000000UL) + return -EINVAL; + } else { + if (addrt[i] & __UA_LIMIT) + return -EINVAL; + } +#endif + __get_user(maskt[i], &addr->WATCH_STYLE[i].mask); + __get_user(irwt[i], &addr->WATCH_STYLE[i].irw); + } + /* Install them. */ + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + if (irwt[i] & LA_WATCH_IRW) + watch_active = 1; + child->thread.watch.addr[i] = addrt[i]; + child->thread.watch.mask[i] = maskt[i]; + child->thread.watch.irw[i] = irwt[i]; + } + + if (watch_active) + set_tsk_thread_flag(child, TIF_LOAD_WATCH); + else + clear_tsk_thread_flag(child, TIF_LOAD_WATCH); + + return 0; +} + + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + int ret; + void __user *addrp = (void __user *) addr; + unsigned long __user *datap = (void __user *) data; + + switch (request) { + case PTRACE_PEEKUSR: + ret = read_user(child, addr, datap); + break; + + case PTRACE_POKEUSR: + ret = write_user(child, addr, data); + break; + + case PTRACE_GET_WATCH_REGS: + ret = ptrace_get_watch_regs(child, addrp); + break; + + case PTRACE_SET_WATCH_REGS: + ret = ptrace_set_watch_regs(child, addrp); + break; + + default: + ret = ptrace_request(child, request, addr, data); + break; + } + + return ret; +} + +void user_enable_single_step(struct task_struct *task) +{ + int i = boot_cpu_data.watch_dreg_count; + struct thread_info *ti = task_thread_info(task); + + task->thread.watch.addr[i] = task_pt_regs(task)->csr_era; + task->thread.watch.mask[i] = -1UL; + task->thread.watch.irw[i] = LA_WATCH_I; + task->thread.single_step = task_pt_regs(task)->csr_era; + set_ti_thread_flag(ti, TIF_SINGLESTEP); +} +EXPORT_SYMBOL(user_enable_single_step); + +void user_disable_single_step(struct task_struct *task) +{ + clear_tsk_thread_flag(task, TIF_SINGLESTEP); +} +EXPORT_SYMBOL(user_disable_single_step); diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c new file mode 100644 index 0000000000000000000000000000000000000000..baa3d0d1f28c7360d475a2a6b91d72e8f19cded8 --- /dev/null +++ b/arch/loongarch/kernel/relocate.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Kernel relocation at boot time + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * Authors: Huacai Chen (chenhuacai@loongson.cn) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RELOCATED(x) ((void *)((long)x + offset)) + +extern u32 _relocation_start[]; /* End kernel image / start relocation table */ +extern u32 _relocation_end[]; /* End relocation table */ + +static void __init apply_r_loongarch_32_rel(u32 *loc_new, long offset) +{ + *loc_new += offset; +} + +static void __init apply_r_loongarch_64_rel(u32 *loc_new, long offset) +{ + *(u64 *)loc_new += offset; +} + +/* + * The details about la.abs $r1, x on LoongArch + * + * lu12i.w $r1, 0 + * ori $r1, $r1, 0x0 + * lu32i.d $r1, 0 + * lu52i.d $r1, $r1, 0 + * + * LoongArch use lu12i.w, ori, lu32i.d, lu52i.d to load a 64bit imm. + * lu12i.w load bit31~bit12, ori load bit11~bit0, + * lu32i.d load bit51~bit32, lu52i.d load bit63~bit52 + */ +static void __init apply_r_loongarch_mark_la_rel(u32 *loc_new, long offset) +{ + unsigned long long dest; + union loongarch_instruction *ori, *lu12iw, *lu32id, *lu52id; + + ori = (union loongarch_instruction *)&loc_new[1]; + lu12iw = (union loongarch_instruction *)&loc_new[0]; + lu32id = (union loongarch_instruction *)&loc_new[2]; + lu52id = (union loongarch_instruction *)&loc_new[3]; + + dest = ori->reg2ui12_format.simmediate & 0xfff; + dest |= (lu12iw->reg1i20_format.simmediate & 0xfffff) << 12; + dest |= ((u64)lu32id->reg1i20_format.simmediate & 0xfffff) << 32; + dest |= ((u64)lu52id->reg2i12_format.simmediate & 0xfff) << 52; + dest += offset; + + ori->reg2ui12_format.simmediate = dest & 0xfff; + lu12iw->reg1i20_format.simmediate = (dest >> 12) & 0xfffff; + lu32id->reg1i20_format.simmediate = (dest >> 32) & 0xfffff; + lu52id->reg2i12_format.simmediate = (dest >> 52) & 0xfff; +} + +static int __init do_relocations(void *kbase_new, long offset, u32 *rstart, u32 *rend) +{ + int type; + u32 *r; + u32 *loc_new; + + for (r = rstart; r < rend; r++) { + /* Sentinel for last relocation */ + if (*r == 0) + break; + + type = (*r >> 24) & 0xff; + loc_new = (void *)(kbase_new + ((*r & 0x00ffffff) << 2)); + + switch (type) { + case R_LARCH_32: + apply_r_loongarch_32_rel(loc_new, offset); + break; + case R_LARCH_64: + apply_r_loongarch_64_rel(loc_new, offset); + break; + case R_LARCH_MARK_LA: + apply_r_loongarch_mark_la_rel(loc_new, offset); + break; + default: + pr_err("Unhandled relocation type %d\n", type); + return -ENOEXEC; + } + } + + return 0; +} + +#ifdef CONFIG_RANDOMIZE_BASE + +static inline __init unsigned long rotate_xor(unsigned long hash, + const void *area, size_t size) +{ + size_t i, diff; + const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash)); + + diff = (void *)ptr - area; + if (unlikely(size < diff + sizeof(hash))) + return hash; + + size = ALIGN_DOWN(size - diff, sizeof(hash)); + + for (i = 0; i < size / sizeof(hash); i++) { + /* Rotate by odd number of bits and XOR. */ + hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); + hash ^= ptr[i]; + } + + return hash; +} + +static inline __init unsigned long get_random_boot(void) +{ + unsigned long hash = 0; + unsigned long entropy = random_get_entropy(); + + /* Attempt to create a simple but unpredictable starting entropy. */ + hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); + + /* Add in any runtime entropy we can get */ + hash = rotate_xor(hash, &entropy, sizeof(entropy)); + + return hash; +} + +static int __init nokaslr(char *p) +{ + pr_info("KASLR is disabled.\n"); + + return 0; /* Print a notice and silence the boot warning */ +} +early_param("nokaslr", nokaslr); + +static inline __init bool kaslr_disabled(void) +{ + char *str; + + const char *builtin_cmdline = CONFIG_CMDLINE; + + str = strstr(builtin_cmdline, "nokaslr"); + if (str == builtin_cmdline || + (str > builtin_cmdline && *(str - 1) == ' ')) + return true; + + str = strstr(boot_command_line, "nokaslr"); + if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) + return true; + + return false; +} + +static inline void __init *determine_relocation_address(void) +{ + /* Choose a new address for the kernel */ + unsigned long kernel_length; + void *dest = &_text; + unsigned long offset; + + if (kaslr_disabled()) + return dest; + + kernel_length = (long)_end - (long)(&_text); + + offset = get_random_boot() << 16; + offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); + if (offset < kernel_length) + offset += ALIGN(kernel_length, 0xffff); + + return RELOCATED(dest); +} + +#else + +static inline void __init *determine_relocation_address(void) +{ + /* + * Choose a new address for the kernel + * For now we'll hard code the destination + */ + return (void *)(CACHE_BASE + 0x02000000); +} + +#endif + +static inline int __init relocation_addr_valid(void *loc_new) +{ + if ((unsigned long)loc_new & 0x00000ffff) + return 0; /* Inappropriately aligned new location */ + + if ((unsigned long)loc_new < (unsigned long)_end) + return 0; /* New location overlaps original kernel */ + + return 1; +} + +static inline void __init update_kaslr_offset(unsigned long *addr, long offset) +{ + unsigned long *new_addr = (unsigned long *)RELOCATED(addr); + + *new_addr = (unsigned long)offset; +} + +void *__init relocate_kernel(void) +{ + void *loc_new; + unsigned long kernel_length; + unsigned long bss_length; + long offset = 0; + int res = 1; + /* Default to original kernel entry point */ + void *kernel_entry = start_kernel; + + early_init_dt_scan(early_ioremap(fw_arg1, SZ_64K)); + + kernel_length = (long)(&_relocation_start) - (long)(&_text); + bss_length = (long)&__bss_stop - (long)&__bss_start; + + loc_new = determine_relocation_address(); + + /* Sanity check relocation address */ + if (relocation_addr_valid(loc_new)) + offset = (unsigned long)loc_new - (unsigned long)(&_text); + + if (offset) { + /* Copy the kernel to it's new location */ + memcpy(loc_new, &_text, kernel_length); + + /* Perform relocations on the new kernel */ + res = do_relocations(loc_new, offset, _relocation_start, _relocation_end); + if (res < 0) + goto out; + + /* Sync the caches ready for execution of new kernel */ + asm volatile ( + " ibar 0 \n" + " dbar 0 \n"); + + /* + * The original .bss has already been cleared, and + * some variables such as command line parameters + * stored to it so make a copy in the new location. + */ + memcpy(RELOCATED(&__bss_start), &__bss_start, bss_length); + + /* The current thread is now within the relocated image */ + __current_thread_info = RELOCATED(__current_thread_info); + + /* Return the new kernel's entry point */ + kernel_entry = RELOCATED(start_kernel); + + update_kaslr_offset(&__kaslr_offset, offset); + } +out: + return kernel_entry; +} + +void *__init relocate_kdump_kernel(long offset) +{ +#ifdef CONFIG_CRASH_DUMP + void *loc_new; + int res = 1; + u32 *rstart, *rend; + /* Default to original kernel entry point */ + void *kernel_entry = start_kernel; + + rstart = RELOCATED((unsigned long)_relocation_start - (unsigned long)&_text + + VMLINUX_LOAD_ADDRESS); + rend = RELOCATED((unsigned long)_relocation_end - (unsigned long)&_text + + VMLINUX_LOAD_ADDRESS); + + loc_new = (void *)(offset + VMLINUX_LOAD_ADDRESS); + + /* Perform relocations on the new kernel */ + res = do_relocations(loc_new, offset, rstart, rend); + if (res < 0) + return NULL; + + /* Return the new kernel's entry point */ + kernel_entry = RELOCATED((unsigned long)start_kernel - (unsigned long)&_text + + VMLINUX_LOAD_ADDRESS); + + return kernel_entry; +#endif + return NULL; +} + +/* + * Show relocation information on panic. + */ +void show_kernel_relocation(const char *level) +{ + if (__kaslr_offset > 0) { + printk(level); + pr_cont("Kernel relocated by 0x%pK\n", (void *)__kaslr_offset); + pr_cont(" .text @ 0x%pK\n", _text); + pr_cont(" .data @ 0x%pK\n", _sdata); + pr_cont(" .bss @ 0x%pK\n", __bss_start); + } +} + +static int kernel_location_notifier_fn(struct notifier_block *self, + unsigned long v, void *p) +{ + show_kernel_relocation(KERN_EMERG); + return NOTIFY_DONE; +} + +static struct notifier_block kernel_location_notifier = { + .notifier_call = kernel_location_notifier_fn +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_location_notifier); + return 0; +} +__initcall(register_kernel_offset_dumper); diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S new file mode 100644 index 0000000000000000000000000000000000000000..358f0b0f38ba9f07d0d5d7c2912c9829715f2677 --- /dev/null +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * relocate_kernel.S for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#include + +#include +#include +#include +#include +#include +#include + +SYM_CODE_START(relocate_new_kernel) + /* + * a0: EFI boot flag for the new kernel + * a1: Command line pointer for the new kernel + * a2: System table pointer for the new kernel + * a3: Start address to jump to after relocation + * a4: Pointer to the current indirection page entry + * a5: Kdump kernel relocate offset + */ + move s0, a4 + + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beqz s0, done + +process_entry: + PTR_L s1, s0, 0 + PTR_ADDI s0, s0, SZREG + + /* destination page */ + andi s2, s1, IND_DESTINATION + beqz s2, 1f + li.w t0, ~0x1 + and s3, s1, t0 /* store destination addr in s3 */ + b process_entry + +1: + /* indirection page, update s0 */ + andi s2, s1, IND_INDIRECTION + beqz s2, 1f + li.w t0, ~0x2 + and s0, s1, t0 + b process_entry + +1: + /* done page */ + andi s2, s1, IND_DONE + beqz s2, 1f + b done + +1: + /* source page */ + andi s2, s1, IND_SOURCE + beqz s2, process_entry + li.w t0, ~0x8 + and s1, s1, t0 + li.w s5, (1 << _PAGE_SHIFT) / SZREG + +copy_word: + /* copy page word by word */ + REG_L s4, s1, 0 + REG_S s4, s3, 0 + PTR_ADDI s3, s3, SZREG + PTR_ADDI s1, s1, SZREG + LONG_ADDI s5, s5, -1 + beqz s5, process_entry + b copy_word + +done: + ibar 0 + dbar 0 + + /* + * Jump to the new kernel, + * make sure the values of a0, a1, a2, a3 and a5 are not changed. + */ + move t0, a3 + move a3, a5 + jr t0 +SYM_CODE_END(relocate_new_kernel) + +#ifdef CONFIG_SMP +/* + * Other CPUs should wait until code is relocated and + * then start at the entry point from LOONGARCH_IOCSR_MBUF0. + */ +SYM_CODE_START(kexec_smp_wait) +1: li.w t0, 0x100 /* wait for init loop */ +2: addi.w t0, t0, -1 /* limit mailbox access */ + bnez t0, 2b + li.w t1, LOONGARCH_IOCSR_MBUF0 + iocsrrd.w s0, t1 /* check PC as an indicator */ + beqz s0, 1b + iocsrrd.d s0, t1 /* get PC via mailbox */ + + li.d t0, CACHE_BASE + or s0, s0, t0 /* s0 = TO_CACHE(s0) */ + jr s0 /* jump to initial PC */ +SYM_CODE_END(kexec_smp_wait) +#endif + +relocate_new_kernel_end: + +SYM_DATA_START(relocate_new_kernel_size) + PTR relocate_new_kernel_end - relocate_new_kernel +SYM_DATA_END(relocate_new_kernel_size) diff --git a/arch/loongarch/kernel/reset.c b/arch/loongarch/kernel/reset.c new file mode 100644 index 0000000000000000000000000000000000000000..2b83631a7aaea143e81b9198b827bbc141cfe36b --- /dev/null +++ b/arch/loongarch/kernel/reset.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static void machine_hang(void) +{ + local_irq_disable(); + clear_csr_ecfg(ECFG0_IM); + + pr_notice("\n\n** You can safely turn off the power now **\n\n"); + console_flush_on_panic(CONSOLE_FLUSH_PENDING); + + while (true) { + __arch_cpu_idle(); + local_irq_disable(); + } +} + +void (*pm_restart)(void) = machine_hang; +void (*pm_power_off)(void) = machine_hang; + +EXPORT_SYMBOL(pm_power_off); + +void machine_halt(void) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + machine_hang(); +} + +void machine_power_off(void) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif +#ifdef CONFIG_PM + enable_pci_wakeup(); +#endif + pm_power_off(); +} + +void machine_restart(char *command) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + do_kernel_restart(command); + pm_restart(); +} diff --git a/arch/loongarch/kernel/scall-trans-i386.S b/arch/loongarch/kernel/scall-trans-i386.S new file mode 100644 index 0000000000000000000000000000000000000000..c3a5ce660c6c26533a68d051ca25db5077182362 --- /dev/null +++ b/arch/loongarch/kernel/scall-trans-i386.S @@ -0,0 +1,495 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1995, 96, 97, 98, 99, 2000, 01, 02 by Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2001 MIPS Technologies, Inc. + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .align 5 +SYM_FUNC_START(handle_sys_lat_i386) + UNWIND_HINT_REGS + li.d t1, 0 #_TIF_WORK_SYSCALL_ENTRY + LONG_L t0, tp, TI_FLAGS # syscall tracing enabled? + and t0, t1, t0 + bnez t0, syscall_trace_entry + +choose_abi: + li.d t0, TRANS_ARCH_MASK + and t1, t0, a7 # pick the ARCH specfied tag + li.d t0, SYS_NUM_MASK + and a7, a7, t0 # pick the syscall num + li.d t0, TRANS_I386 + bne t0, t1, illegal_syscall + +syscall_i386: + + sltui t0, a7, __NR_i386_Linux_syscalls + 1 + beqz t0, illegal_syscall + + /* Syscall number held in a7 */ + slli.d t0, a7, 3 # offset into table + la t2, i386_syscall_table + add.d t0, t2, t0 + ld.d t2, t0, 0 # syscall routine + beqz t2, illegal_syscall + + jalr t2 # Do The Real Thing (TM) + + li.w t0, -EMAXERRNO - 1 # error? + sltu t0, t0, a0 + st.d t0, sp, PT_R7 # set error flag + beqz t0, 1f + + ld.d t1, sp, PT_R11 # syscall number + addi.d t1, t1, 1 # +1 for handle_signal + st.d t1, sp, PT_R0 # save it for syscall restarting +1: st.d a0, sp, PT_R4 # result + + +trans_i386_syscall_exit: + NOT_SIBLING_CALL_HINT + RESTORE_TEMP + RESTORE_STATIC + RESTORE_SOME + RESTORE_SP_AND_RET + +/* ------------------------------------------------------------------------ */ + +syscall_trace_entry: + SAVE_STATIC + move a0, sp + move a1, a7 + move a0, zero #bl syscall_trace_enter + + blt a0, zero, 1f # seccomp failed? Skip syscall + + RESTORE_STATIC + ld.d a0, sp, PT_R4 # Restore argument registers + ld.d a1, sp, PT_R5 + ld.d a2, sp, PT_R6 + ld.d a3, sp, PT_R7 + ld.d a4, sp, PT_R8 + ld.d a5, sp, PT_R9 + ld.d a6, sp, PT_R10 + ld.d a7, sp, PT_R11 # Restore syscall (maybe modified) + b choose_abi + +1: b trans_i386_syscall_exit + + /* + * The system call does not exist in this kernel + */ + +illegal_syscall: + li.w a0, ENOSYS # error + st.d a0, sp, PT_R4 + li.w t0, 1 # set error flag + st.d t0, sp, PT_R7 + b trans_i386_syscall_exit +SYM_FUNC_END(handle_sys_lat_i386) + + .align 3 +SYM_DATA_START(i386_syscall_table) + PTR sys_restart_syscall /* 0 */ + PTR sys_exit /* 1 */ + PTR sys_ni_syscall /* 2 sys_fork */ + PTR sys_read /* 3 */ + PTR sys_write /* 4 */ + PTR sys_open /* 5 */ + PTR sys_close /* 6 */ + PTR sys_ni_syscall /* 7 sys_waitpid */ + PTR sys_creat /* 8 */ + PTR sys_link /* 9 */ + PTR sys_unlink /* 10 */ + PTR sys_execve /* 11 */ + PTR sys_chdir /* 12 */ + PTR sys_ni_syscall /* 13 sys_time */ + PTR sys_mknod /* 14 */ + PTR sys_chmod /* 15 */ + PTR sys_lchown16 /* 16 */ + PTR sys_ni_syscall /* 17 break */ + PTR sys_ni_syscall /* 18 sys_stat */ + PTR sys_lseek /* 19 */ + PTR sys_getpid /* 20 */ + PTR sys_mount /* 21 */ + PTR sys_ni_syscall /* 22 sys_oldumount */ + PTR sys_setuid16 /* 23 */ + PTR sys_getuid16 /* 24 */ + PTR sys_ni_syscall /* 25 sys_stime */ + PTR sys_ptrace /* 26 */ + PTR sys_alarm /* 27 */ + PTR sys_ni_syscall /* 28 sys_fstat */ + PTR sys_ni_syscall /* 29 sys_pause */ + PTR sys_ni_syscall /* 30 sys_utime */ + PTR sys_ni_syscall /* 31 stty */ + PTR sys_ni_syscall /* 32 gtty */ + PTR sys_access /* 33 */ + PTR sys_ni_syscall /* 34 sys_nice */ + PTR sys_ni_syscall /* 35 ftime */ + PTR sys_sync /* 36 */ + PTR sys_kill /* 37 */ + PTR sys_rename /* 38 */ + PTR sys_mkdir /* 39 */ + PTR sys_rmdir /* 40 */ + PTR sys_dup /* 41 */ + PTR sys_pipe /* 42 */ + PTR sys_times /* 43 */ + PTR sys_ni_syscall /* 44 prof */ + PTR sys_brk /* 45 */ + PTR sys_setgid16 /* 46 */ + PTR sys_getgid16 /* 47 */ + PTR sys_ni_syscall /* 48 sys_signal */ + PTR sys_geteuid16 /* 49 */ + PTR sys_getegid16 /* 50 */ + PTR sys_acct /* 51 */ + PTR sys_umount /* 52 */ + PTR sys_ni_syscall /* 53 lock */ + PTR sys_ioctl /* 54 */ + PTR sys_fcntl /* 55 */ + PTR sys_ni_syscall /* 56 mxp */ + PTR sys_setpgid /* 57 */ + PTR sys_ni_syscall /* 58 ulimit */ + PTR sys_ni_syscall /* 59 sys_olduname */ + PTR sys_umask /* 60 */ + PTR sys_chroot /* 61 */ + PTR sys_ustat /* 62 */ + PTR sys_dup2 /* 63 */ + PTR sys_getppid /* 64 */ + PTR sys_getpgrp /* 65 */ + PTR sys_setsid /* 66 */ + PTR sys_ni_syscall /* 67 sys_sigaction */ + PTR sys_sgetmask /* 68 */ + PTR sys_ssetmask /* 69 */ + PTR sys_setreuid16 /* 70 */ + PTR sys_setregid16 /* 71 */ + PTR sys_ni_syscall /* 72 sys_sigsuspend */ + PTR sys_ni_syscall /* 73 sys_sigpending */ + PTR sys_sethostname /* 74 */ + PTR sys_setrlimit /* 75 */ + PTR sys_ni_syscall /* 76 sys_old_getrlimit */ + PTR sys_getrusage /* 77 */ + PTR sys_gettimeofday /* 78 */ + PTR sys_settimeofday /* 79 */ + PTR sys_getgroups16 /* 80 */ + PTR sys_setgroups16 /* 81 */ + PTR sys_ni_syscall /* 82 sys_old_select */ + PTR sys_symlink /* 83 */ + PTR sys_ni_syscall /* 84 sys_lstat */ + PTR sys_readlink /* 85 */ + PTR sys_uselib /* 86 */ + PTR sys_swapon /* 87 */ + PTR sys_reboot /* 88 */ + PTR sys_ni_syscall /* 89 sys_old_readdir */ + PTR sys_ni_syscall /* 90 sys_old_mmap */ + PTR sys_munmap /* 91 */ + PTR sys_truncate /* 92 */ + PTR sys_ftruncate /* 93 */ + PTR sys_fchmod /* 94 */ + PTR sys_fchown16 /* 95 */ + PTR sys_getpriority /* 96 */ + PTR sys_setpriority /* 97 */ + PTR sys_ni_syscall /* 98 profil */ + PTR sys_statfs /* 99 */ + PTR sys_fstatfs /* 100 */ + PTR sys_ni_syscall /* 101 sys_ioperm */ + PTR sys_socketcall /* 102 */ + PTR sys_syslog /* 103 */ + PTR sys_setitimer /* 104 */ + PTR sys_getitimer /* 105 */ + PTR sys_newstat /* 106 */ + PTR sys_newlstat /* 107 */ + PTR sys_newfstat /* 108 */ + PTR sys_ni_syscall /* 109 sys_uname */ + PTR sys_ni_syscall /* 110 sys_iopl */ + PTR sys_vhangup /* 111 */ + PTR sys_ni_syscall /* 112 idel */ + PTR sys_vm86old /* 113 */ + PTR sys_wait4 /* 114 */ + PTR sys_swapoff /* 115 */ + PTR sys_sysinfo /* 116 */ + PTR sys_ipc /* 117 */ + PTR sys_fsync /* 118 */ + PTR sys_ni_syscall /* 119 sys_sigreturn */ + PTR sys_clone /* 120 */ + PTR sys_setdomainname /* 121 */ + PTR sys_newuname /* 122 */ + PTR sys_modify_ldt /* 123 */ + PTR sys_adjtimex /* 124 */ + PTR sys_mprotect /* 125 */ + PTR sys_ni_syscall /* 126 sys_sigprocmask */ + PTR sys_ni_syscall /* 127 create_module */ + PTR sys_init_module /* 128 */ + PTR sys_delete_module /* 129 */ + PTR sys_ni_syscall /* 130 get_kernel_syms */ + PTR sys_quotactl /* 131 */ + PTR sys_getpgid /* 132 */ + PTR sys_fchdir /* 133 */ + PTR sys_bdflush /* 134 */ + PTR sys_sysfs /* 135 */ + PTR sys_personality /* 136 */ + PTR sys_ni_syscall /* 137 afs_syscall */ + PTR sys_setfsuid16 /* 138 */ + PTR sys_setfsgid16 /* 139 */ + PTR sys_ni_syscall /* 140 sys_llseek */ + PTR sys_getdents /* 141 */ + PTR sys_select /* 142 */ + PTR sys_flock /* 143 */ + PTR sys_msync /* 144 */ + PTR sys_readv /* 145 */ + PTR sys_writev /* 146 */ + PTR sys_getsid /* 147 */ + PTR sys_fdatasync /* 148 */ + PTR sys_ni_syscall //sys_sysctl /* 149 */ + PTR sys_mlock /* 150 */ + PTR sys_munlock /* 151 */ + PTR sys_mlockall /* 152 */ + PTR sys_munlockall /* 153 */ + PTR sys_sched_setparam /* 154 */ + PTR sys_sched_getparam /* 155 */ + PTR sys_sched_setscheduler /* 156 */ + PTR sys_sched_getscheduler /* 157 */ + PTR sys_sched_yield /* 158 */ + PTR sys_sched_get_priority_max /* 159 */ + PTR sys_sched_get_priority_min /* 160 */ + PTR sys_sched_rr_get_interval /* 161 */ + PTR sys_nanosleep /* 162 */ + PTR sys_mremap /* 163 */ + PTR sys_setresuid16 /* 164 */ + PTR sys_getresuid16 /* 165 */ + PTR sys_vm86 /* 166 */ + PTR sys_ni_syscall /* 167 query_module */ + PTR sys_poll /* 168 */ + PTR sys_ni_syscall /* 169 nfsservctl */ + PTR sys_setresgid16 /* 170 */ + PTR sys_getresgid16 /* 171 */ + PTR sys_prctl /* 172 */ + PTR sys_rt_sigreturn /* 173 */ + PTR sys_rt_sigaction /* 174 */ + PTR sys_latx_rt_sigprocmask /* 175 */ + PTR sys_rt_sigpending /* 176 */ + PTR sys_rt_sigtimedwait /* 177 */ + PTR sys_rt_sigqueueinfo /* 178 */ + PTR sys_rt_sigsuspend /* 179 */ + PTR sys_pread64 /* 180 */ + PTR sys_pwrite64 /* 181 */ + PTR sys_chown16 /* 182 */ + PTR sys_getcwd /* 183 */ + PTR sys_capget /* 184 */ + PTR sys_capset /* 185 */ + PTR sys_sigaltstack /* 186 */ + PTR sys_sendfile /* 187 */ + PTR sys_ni_syscall /* 188 getpmsg */ + PTR sys_ni_syscall /* 189 putpmsg */ + PTR sys_ni_syscall /* 190 sys_vfork */ + PTR sys_getrlimit /* 191 */ + PTR sys_mmap_pgoff /* 192 */ + PTR sys_ni_syscall /* 193 sys_truncate64 */ + PTR sys_ni_syscall /* 194 sys_ftruncate64 */ + PTR sys_ni_syscall /* 195 sys_stat64 */ + PTR sys_ni_syscall /* 196 sys_lstat64 */ + PTR sys_ni_syscall /* 197 sys_fstat64 */ + PTR sys_lchown /* 198 */ + PTR sys_getuid /* 199 */ + PTR sys_getgid /* 200 */ + PTR sys_geteuid /* 201 */ + PTR sys_getegid /* 202 */ + PTR sys_setreuid /* 203 */ + PTR sys_setregid /* 204 */ + PTR sys_getgroups /* 205 */ + PTR sys_setgroups /* 206 */ + PTR sys_fchown /* 207 */ + PTR sys_setresuid /* 208 */ + PTR sys_getresuid /* 209 */ + PTR sys_setresgid /* 210 */ + PTR sys_getresgid /* 211 */ + PTR sys_chown /* 212 */ + PTR sys_setuid /* 213 */ + PTR sys_setgid /* 214 */ + PTR sys_setfsuid /* 215 */ + PTR sys_setfsgid /* 216 */ + PTR sys_pivot_root /* 217 */ + PTR sys_mincore /* 218 */ + PTR sys_madvise /* 219 */ + PTR sys_getdents64 /* 220 */ + PTR sys_ni_syscall /* 221 sys_fcntl64 */ + PTR sys_ni_syscall /* 222 is unused */ + PTR sys_ni_syscall /* 223 is unused */ + PTR sys_gettid /* 224 */ + PTR sys_readahead /* 225 */ + PTR sys_setxattr /* 226 */ + PTR sys_lsetxattr /* 227 */ + PTR sys_fsetxattr /* 228 */ + PTR sys_getxattr /* 229 */ + PTR sys_lgetxattr /* 230 */ + PTR sys_fgetxattr /* 231 */ + PTR sys_listxattr /* 232 */ + PTR sys_llistxattr /* 233 */ + PTR sys_flistxattr /* 234 */ + PTR sys_removexattr /* 235 */ + PTR sys_lremovexattr /* 236 */ + PTR sys_fremovexattr /* 237 */ + PTR sys_tkill /* 238 */ + PTR sys_sendfile64 /* 239 */ + PTR sys_futex /* 240 */ + PTR sys_sched_setaffinity /* 241 */ + PTR sys_sched_getaffinity /* 242 */ + PTR sys_ni_syscall /* 243 sys_set_thread_area */ + PTR sys_ni_syscall /* 244 sys_get_thread_area */ + PTR sys_io_setup /* 245 */ + PTR sys_io_destroy /* 246 */ + PTR sys_io_getevents /* 247 */ + PTR sys_io_submit /* 248 */ + PTR sys_io_cancel /* 249 */ + PTR sys_fadvise64 /* 250 */ + PTR sys_ni_syscall /* 251 is available for reuse*/ + PTR sys_exit_group /* 252 */ + PTR sys_lookup_dcookie /* 253 */ + PTR sys_epoll_create /* 254 */ + PTR sys_epoll_ctl /* 255 */ + PTR sys_epoll_wait /* 256 */ + PTR sys_remap_file_pages /* 257 */ + PTR sys_set_tid_address /* 258 */ + PTR sys_timer_create /* 259 */ + PTR sys_timer_settime /* 260 */ + PTR sys_timer_gettime /* 261 */ + PTR sys_timer_getoverrun /* 262 */ + PTR sys_timer_delete /* 263 */ + PTR sys_clock_settime /* 264 */ + PTR sys_clock_gettime /* 265 */ + PTR sys_clock_getres /* 266 */ + PTR sys_clock_nanosleep /* 267 */ + PTR sys_statfs64 /* 268 */ + PTR sys_fstatfs64 /* 269 */ + PTR sys_tgkill /* 270 */ + PTR sys_ni_syscall //sys_utimes /* 271 */ + PTR sys_fadvise64_64 /* 272 */ + PTR sys_ni_syscall /* 273 vserver */ + PTR sys_mbind /* 274 */ + PTR sys_get_mempolicy /* 275 */ + PTR sys_set_mempolicy /* 276 */ + PTR sys_mq_open /* 277 */ + PTR sys_mq_unlink /* 278 */ + PTR sys_mq_timedsend /* 279 */ + PTR sys_mq_timedreceive /* 280 */ + PTR sys_mq_notify /* 281 */ + PTR sys_mq_getsetattr /* 282 */ + PTR sys_kexec_load /* 283 */ + PTR sys_waitid /* 284 */ + PTR sys_ni_syscall /* 285 sys_setaltroot */ + PTR sys_add_key /* 286 */ + PTR sys_request_key /* 287 */ + PTR sys_keyctl /* 288 */ + PTR sys_ioprio_set /* 289 */ + PTR sys_ioprio_get /* 290 */ + PTR sys_inotify_init /* 291 */ + PTR sys_inotify_add_watch /* 292 */ + PTR sys_inotify_rm_watch /* 293 */ + PTR sys_migrate_pages /* 294 */ + PTR sys_openat /* 295 */ + PTR sys_mkdirat /* 296 */ + PTR sys_mknodat /* 297 */ + PTR sys_fchownat /* 298 */ + PTR sys_ni_syscall //sys_futimesat /* 299 */ + PTR sys_ni_syscall /* 300 sys_fstatat64 */ + PTR sys_unlinkat /* 301 */ + PTR sys_renameat /* 302 */ + PTR sys_linkat /* 303 */ + PTR sys_symlinkat /* 304 */ + PTR sys_readlinkat /* 305 */ + PTR sys_fchmodat /* 306 */ + PTR sys_faccessat /* 307 */ + PTR sys_pselect6 /* 308 */ + PTR sys_ppoll /* 309 */ + PTR sys_unshare /* 310 */ + PTR sys_set_robust_list /* 311 */ + PTR sys_get_robust_list /* 312 */ + PTR sys_splice /* 313 */ + PTR sys_sync_file_range /* 314 */ + PTR sys_tee /* 315 */ + PTR sys_vmsplice /* 316 */ + PTR sys_move_pages /* 317 */ + PTR sys_getcpu /* 318 */ + PTR sys_epoll_pwait /* 319 */ + PTR sys_utimensat /* 320 */ + PTR sys_signalfd /* 321 */ + PTR sys_timerfd_create /* 322 */ + PTR sys_eventfd /* 323 */ + PTR sys_fallocate /* 324 */ + PTR sys_timerfd_settime /* 325 */ + PTR sys_timerfd_gettime /* 326 */ + PTR sys_signalfd4 /* 327 */ + PTR sys_eventfd2 /* 328 */ + PTR sys_epoll_create1 /* 329 */ + PTR sys_dup3 /* 330 */ + PTR sys_pipe2 /* 331 */ + PTR sys_inotify_init1 /* 332 */ + PTR sys_preadv /* 333 */ + PTR sys_pwritev /* 334 */ + PTR sys_rt_tgsigqueueinfo /* 335 */ + PTR sys_perf_event_open /* 336 */ + PTR sys_recvmmsg /* 337 */ + PTR sys_fanotify_init /* 338 */ + PTR sys_fanotify_mark /* 339 */ + PTR sys_prlimit64 /* 340 */ + PTR sys_name_to_handle_at /* 341 */ + PTR sys_open_by_handle_at /* 342 */ + PTR sys_clock_adjtime /* 343 */ + PTR sys_syncfs /* 344 */ + PTR sys_sendmmsg /* 345 */ + PTR sys_setns /* 346 */ + PTR sys_process_vm_readv /* 347 */ + PTR sys_process_vm_writev /* 348 */ + PTR sys_kcmp /* 349 */ + PTR sys_finit_module /* 350 */ + PTR sys_sched_setattr /* 351 */ + PTR sys_sched_getattr /* 352 */ + PTR sys_renameat2 /* 353 */ + PTR sys_seccomp /* 354 */ + PTR sys_getrandom /* 355 */ + PTR sys_memfd_create /* 356 */ + PTR sys_bpf /* 357 */ + PTR sys_execveat /* 358 */ + PTR sys_socket /* 359 */ + PTR sys_socketpair /* 360 */ + PTR sys_bind /* 361 */ + PTR sys_connect /* 362 */ + PTR sys_listen /* 363 */ + PTR sys_accept4 /* 364 */ + PTR sys_getsockopt /* 365 */ + PTR sys_setsockopt /* 366 */ + PTR sys_getsockname /* 367 */ + PTR sys_getpeername /* 368 */ + PTR sys_sendto /* 369 */ + PTR sys_sendmsg /* 370 */ + PTR sys_recvfrom /* 371 */ + PTR sys_recvmsg /* 372 */ + PTR sys_shutdown /* 373 */ + PTR sys_userfaultfd /* 374 */ + PTR sys_membarrier /* 375 */ + PTR sys_mlock2 /* 376 */ + PTR sys_copy_file_range /* 377 */ + PTR sys_preadv2 /* 378 */ + PTR sys_pwritev2 /* 379 */ + PTR sys_pkey_mprotect /* 380 */ + PTR sys_pkey_alloc /* 381 */ + PTR sys_pkey_free /* 382 */ + PTR sys_statx /* 383 */ + PTR sys_ni_syscall /* 384 sys_arch_prctl */ + PTR sys_io_pgetevents /* 385 */ + PTR sys_rseq /* 386 */ +SYM_DATA_END(i386_syscall_table) diff --git a/arch/loongarch/kernel/scall-trans-mips64.S b/arch/loongarch/kernel/scall-trans-mips64.S new file mode 100644 index 0000000000000000000000000000000000000000..a1a32ee247f9a5577ab6cca6bf59422021153dc9 --- /dev/null +++ b/arch/loongarch/kernel/scall-trans-mips64.S @@ -0,0 +1,437 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .align 5 +SYM_FUNC_START(handle_sys_lat_mips64) + UNWIND_HINT_REGS + li.d t1, 0 #_TIF_WORK_SYSCALL_ENTRY + LONG_L t0, tp, TI_FLAGS # syscall tracing enabled? + and t0, t1, t0 + bnez t0, syscall_trace_entry + +choose_abi: + li.d t0, TRANS_ARCH_MASK + and t1, t0, a7 + li.d t0, SYS_NUM_MASK + and a7, a7, t0 + li.d t0, TRANS_MIPS_N64 + bne t0, t1, illegal_syscall + +syscall_mips64: + li.w t0, __NR_MIPS64_Linux + sub.d t2, a7, t0 + sltui t0, t2, __NR_MIPS64_Linux_syscalls + 1 + beqz t0, illegal_syscall + + /* Syscall number held in a7 */ + slli.d t0, t2, 3 # offset into table + la t2, mips64_syscall_table + add.d t0, t2, t0 + ld.d t2, t0, 0 # syscall routine + beqz t2, illegal_syscall + + jalr t2 # Do The Real Thing (TM) + + li.w t0, -EMAXERRNO - 1 # error? + sltu t0, t0, a0 + st.d t0, sp, PT_R7 # set error flag + beqz t0, 1f + + ld.d t1, sp, PT_R11 # syscall number + addi.d t1, t1, 1 # +1 for handle_signal + st.d t1, sp, PT_R0 # save it for syscall restarting + sub.d a0, zero, a0 + bl trans_mips_errno +1: st.d a0, sp, PT_R4 # result + + +trans_syscall_exit: + NOT_SIBLING_CALL_HINT + RESTORE_TEMP + RESTORE_STATIC + RESTORE_SOME + RESTORE_SP_AND_RET + +/* ------------------------------------------------------------------------ */ + +syscall_trace_entry: + SAVE_STATIC + move a0, sp + move a1, a7 + move a0, zero #bl syscall_trace_enter + + blt a0, zero, 1f # seccomp failed? Skip syscall + + RESTORE_STATIC + ld.d a0, sp, PT_R4 # Restore argument registers + ld.d a1, sp, PT_R5 + ld.d a2, sp, PT_R6 + ld.d a3, sp, PT_R7 + ld.d a4, sp, PT_R8 + ld.d a5, sp, PT_R9 + ld.d a6, sp, PT_R10 + ld.d a7, sp, PT_R11 # Restore syscall (maybe modified) + b choose_abi + +1: b trans_syscall_exit + + /* + * The system call does not exist in this kernel + */ + +illegal_syscall: + li.w a0, ENOSYS # error + st.d a0, sp, PT_R4 + li.w t0, 1 # set error flag + st.d t0, sp, PT_R7 + b trans_syscall_exit +SYM_FUNC_END(handle_sys_lat_mips64) + + .align 3 +SYM_DATA_START(mips64_syscall_table) + PTR sys_read /* 5000 */ + PTR sys_write + PTR sys_ni_syscall + PTR sys_close + PTR sys_ni_syscall /* stat */ + PTR sys_ni_syscall /* 5005 */ + PTR sys_ni_syscall /* lstat */ + PTR sys_ni_syscall + PTR sys_lseek + PTR sys_ni_syscall + PTR sys_mprotect /* 5010 */ + PTR sys_munmap + PTR sys_brk + PTR sys_ni_syscall + PTR sys_rt_sigprocmask + PTR sys_ioctl /* 5015 */ + PTR sys_pread64 + PTR sys_pwrite64 + PTR sys_readv + PTR sys_writev + PTR sys_access /* 5020 */ + PTR sys_pipe + PTR sys_ni_syscall + PTR sys_sched_yield + PTR sys_mremap + PTR sys_msync /* 5025 */ + PTR sys_mincore + PTR sys_madvise + PTR sys_shmget + PTR sys_shmat + PTR sys_shmctl /* 5030 */ + PTR sys_dup + PTR sys_dup2 + PTR sys_ni_syscall /* pause */ + PTR sys_nanosleep + PTR sys_getitimer /* 5035 */ + PTR sys_setitimer + PTR sys_alarm + PTR sys_getpid + PTR sys_sendfile64 + PTR sys_ni_syscall /* 5040 */ + PTR sys_connect + PTR sys_accept + PTR sys_sendto + PTR sys_recvfrom + PTR sys_sendmsg /* 5045 */ + PTR sys_recvmsg + PTR sys_shutdown + PTR sys_bind + PTR sys_listen + PTR sys_getsockname /* 5050 */ + PTR sys_getpeername + PTR sys_socketpair + PTR sys_setsockopt + PTR sys_getsockopt + PTR sys_ni_syscall /* 5055 */ + PTR sys_ni_syscall + PTR sys_execve + PTR sys_exit + PTR sys_wait4 + PTR sys_kill /* 5060 */ + PTR sys_newuname + PTR sys_semget + PTR sys_semop + PTR sys_semctl + PTR sys_shmdt /* 5065 */ + PTR sys_msgget + PTR sys_msgsnd + PTR sys_msgrcv + PTR sys_msgctl + PTR sys_fcntl /* 5070 */ + PTR sys_flock + PTR sys_fsync + PTR sys_fdatasync + PTR sys_truncate + PTR sys_ftruncate /* 5075 */ + PTR sys_ni_syscall + PTR sys_getcwd + PTR sys_chdir + PTR sys_fchdir + PTR sys_rename /* 5080 */ + PTR sys_mkdir + PTR sys_rmdir + PTR sys_ni_syscall + PTR sys_link + PTR sys_unlink /* 5085 */ + PTR sys_symlink + PTR sys_readlink + PTR sys_chmod + PTR sys_fchmod + PTR sys_chown /* 5090 */ + PTR sys_fchown + PTR sys_lchown + PTR sys_umask + PTR sys_gettimeofday + PTR sys_getrlimit /* 5095 */ + PTR sys_getrusage + PTR sys_sysinfo + PTR sys_times + PTR sys_ptrace + PTR sys_getuid /* 5100 */ + PTR sys_syslog + PTR sys_getgid + PTR sys_setuid + PTR sys_setgid + PTR sys_geteuid /* 5105 */ + PTR sys_getegid + PTR sys_setpgid + PTR sys_getppid + PTR sys_getpgrp + PTR sys_setsid /* 5110 */ + PTR sys_setreuid + PTR sys_setregid + PTR sys_getgroups + PTR sys_setgroups + PTR sys_setresuid /* 5115 */ + PTR sys_getresuid + PTR sys_setresgid + PTR sys_getresgid + PTR sys_getpgid + PTR sys_setfsuid /* 5120 */ + PTR sys_setfsgid + PTR sys_getsid + PTR sys_capget + PTR sys_capset + PTR sys_rt_sigpending /* 5125 */ + PTR sys_rt_sigtimedwait + PTR sys_rt_sigqueueinfo + PTR sys_rt_sigsuspend + PTR sys_sigaltstack + PTR sys_ni_syscall /* 5130 */ + PTR sys_mknod + PTR sys_personality + PTR sys_ni_syscall + PTR sys_ni_syscall /* statfs */ + PTR sys_ni_syscall /* 5135 */ + PTR sys_ni_syscall + PTR sys_getpriority + PTR sys_setpriority + PTR sys_sched_setparam + PTR sys_sched_getparam /* 5140 */ + PTR sys_sched_setscheduler + PTR sys_sched_getscheduler + PTR sys_sched_get_priority_max + PTR sys_sched_get_priority_min + PTR sys_sched_rr_get_interval /* 5145 */ + PTR sys_mlock + PTR sys_munlock + PTR sys_mlockall + PTR sys_munlockall + PTR sys_vhangup /* 5150 */ + PTR sys_pivot_root + PTR sys_ni_syscall + PTR sys_prctl + PTR sys_adjtimex + PTR sys_setrlimit /* 5155 */ + PTR sys_chroot + PTR sys_sync + PTR sys_acct + PTR sys_settimeofday + PTR sys_mount /* 5160 */ + PTR sys_umount + PTR sys_swapon + PTR sys_swapoff + PTR sys_reboot + PTR sys_sethostname /* 5165 */ + PTR sys_setdomainname + PTR sys_ni_syscall /* was create_module */ + PTR sys_init_module + PTR sys_delete_module + PTR sys_ni_syscall /* 5170, was get_kernel_syms */ + PTR sys_ni_syscall /* was query_module */ + PTR sys_quotactl + PTR sys_ni_syscall /* was nfsservctl */ + PTR sys_ni_syscall /* res. for getpmsg */ + PTR sys_ni_syscall /* 5175 for putpmsg */ + PTR sys_ni_syscall /* res. for afs_syscall */ + PTR sys_ni_syscall /* res. for security */ + PTR sys_gettid + PTR sys_readahead + PTR sys_setxattr /* 5180 */ + PTR sys_lsetxattr + PTR sys_fsetxattr + PTR sys_getxattr + PTR sys_lgetxattr + PTR sys_fgetxattr /* 5185 */ + PTR sys_listxattr + PTR sys_llistxattr + PTR sys_flistxattr + PTR sys_removexattr + PTR sys_lremovexattr /* 5190 */ + PTR sys_fremovexattr + PTR sys_tkill + PTR sys_ni_syscall + PTR sys_futex + PTR sys_sched_setaffinity /* 5195 */ + PTR sys_sched_getaffinity + PTR sys_ni_syscall + PTR sys_ni_syscall + PTR sys_ni_syscall + PTR sys_io_setup /* 5200 */ + PTR sys_io_destroy + PTR sys_io_getevents + PTR sys_io_submit + PTR sys_io_cancel + PTR sys_exit_group /* 5205 */ + PTR sys_lookup_dcookie + PTR sys_epoll_create + PTR sys_epoll_ctl + PTR sys_epoll_wait + PTR sys_remap_file_pages /* 5210 */ + PTR sys_rt_sigreturn + PTR sys_set_tid_address + PTR sys_restart_syscall + PTR sys_semtimedop + PTR sys_fadvise64_64 /* 5215 */ + PTR sys_timer_create + PTR sys_timer_settime + PTR sys_timer_gettime + PTR sys_timer_getoverrun + PTR sys_timer_delete /* 5220 */ + PTR sys_clock_settime + PTR sys_clock_gettime + PTR sys_clock_getres + PTR sys_clock_nanosleep + PTR sys_tgkill /* 5225 */ + PTR sys_ni_syscall /* utimes */ + PTR sys_mbind + PTR sys_get_mempolicy + PTR sys_set_mempolicy + PTR sys_mq_open /* 5230 */ + PTR sys_mq_unlink + PTR sys_mq_timedsend + PTR sys_mq_timedreceive + PTR sys_mq_notify + PTR sys_mq_getsetattr /* 5235 */ + PTR sys_ni_syscall /* sys_vserver */ + PTR sys_waitid + PTR sys_ni_syscall /* available, was setaltroot */ + PTR sys_add_key + PTR sys_request_key /* 5240 */ + PTR sys_keyctl + PTR sys_ni_syscall /* set_thread_area */ + PTR sys_inotify_init + PTR sys_inotify_add_watch + PTR sys_inotify_rm_watch /* 5245 */ + PTR sys_migrate_pages + PTR sys_ni_syscall + PTR sys_mkdirat + PTR sys_mknodat + PTR sys_fchownat /* 5250 */ + PTR sys_ni_syscall /* futimesat */ + PTR sys_ni_syscall /* newfstatat */ + PTR sys_unlinkat + PTR sys_renameat + PTR sys_linkat /* 5255 */ + PTR sys_symlinkat + PTR sys_readlinkat + PTR sys_fchmodat + PTR sys_faccessat + PTR sys_pselect6 /* 5260 */ + PTR sys_ppoll + PTR sys_unshare + PTR sys_splice + PTR sys_sync_file_range + PTR sys_tee /* 5265 */ + PTR sys_vmsplice + PTR sys_move_pages + PTR sys_set_robust_list + PTR sys_get_robust_list + PTR sys_kexec_load /* 5270 */ + PTR sys_getcpu + PTR sys_epoll_pwait + PTR sys_ioprio_set + PTR sys_ioprio_get + PTR sys_utimensat /* 5275 */ + PTR sys_ni_syscall /* signalfd */ + PTR sys_ni_syscall /* was timerfd */ + PTR sys_ni_syscall /* eventfd */ + PTR sys_fallocate + PTR sys_timerfd_create /* 5280 */ + PTR sys_timerfd_gettime + PTR sys_timerfd_settime + PTR sys_signalfd4 + PTR sys_eventfd2 + PTR sys_epoll_create1 /* 5285 */ + PTR sys_dup3 + PTR sys_pipe2 + PTR sys_inotify_init1 + PTR sys_preadv + PTR sys_pwritev /* 5290 */ + PTR sys_rt_tgsigqueueinfo + PTR sys_perf_event_open + PTR sys_accept4 + PTR sys_recvmmsg + PTR sys_fanotify_init /* 5295 */ + PTR sys_fanotify_mark + PTR sys_ni_syscall + PTR sys_name_to_handle_at + PTR sys_open_by_handle_at + PTR sys_clock_adjtime /* 5300 */ + PTR sys_syncfs + PTR sys_sendmmsg + PTR sys_setns + PTR sys_process_vm_readv + PTR sys_process_vm_writev /* 5305 */ + PTR sys_kcmp + PTR sys_finit_module + PTR sys_getdents64 + PTR sys_sched_setattr + PTR sys_sched_getattr /* 5310 */ + PTR sys_renameat2 + PTR sys_seccomp + PTR sys_getrandom + PTR sys_memfd_create + PTR sys_bpf /* 5315 */ + PTR sys_execveat + PTR sys_userfaultfd + PTR sys_membarrier + PTR sys_mlock2 + PTR sys_copy_file_range /* 5320 */ + PTR sys_preadv2 + PTR sys_pwritev2 + PTR sys_pkey_mprotect + PTR sys_pkey_alloc + PTR sys_pkey_free /* 5325 */ + PTR sys_statx + PTR sys_rseq + PTR sys_io_pgetevents + PTR sys_set_user_tp +SYM_DATA_END(mips64_syscall_table) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c new file mode 100644 index 0000000000000000000000000000000000000000..6cf43d79ba8db1075ef30589164bd66a92f653db --- /dev/null +++ b/arch/loongarch/kernel/setup.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_PER_CPU(unsigned long, kernelsp); +unsigned long fw_arg0, fw_arg1, fw_arg2; +struct cpuinfo_loongarch cpu_data[NR_CPUS] __read_mostly; + +EXPORT_SYMBOL(cpu_data); + +#ifdef CONFIG_VT +struct screen_info screen_info; +#endif + +/* + * Setup information + * + * These are initialized so they are in the .data section + */ + +static int num_standard_resources; +static struct resource *standard_resources; + +static struct resource code_resource = { .name = "Kernel code", }; +static struct resource data_resource = { .name = "Kernel data", }; +static struct resource bss_resource = { .name = "Kernel bss", }; + +unsigned long __kaslr_offset __ro_after_init; +EXPORT_SYMBOL(__kaslr_offset); + +void __init arch_cpu_finalize_init(void) +{ + alternative_instructions(); +} + +static void *detect_magic __initdata = detect_memory_region; + +void __init detect_memory_region(phys_addr_t start, phys_addr_t sz_min, phys_addr_t sz_max) +{ + void *dm = &detect_magic; + phys_addr_t size; + + for (size = sz_min; size < sz_max; size <<= 1) { + if (!memcmp(dm, dm + size, sizeof(detect_magic))) + break; + } + + pr_debug("Memory: %lluMB of RAM detected at 0x%llx (min: %lluMB, max: %lluMB)\n", + ((unsigned long long) size) / SZ_1M, + (unsigned long long) start, + ((unsigned long long) sz_min) / SZ_1M, + ((unsigned long long) sz_max) / SZ_1M); + + memblock_add(start, size); +} + +static void __init dt_bootmem_init(void) +{ + phys_addr_t ramstart, ramend; + unsigned long start, end; + int i; + + ramstart = memblock_start_of_DRAM(); + ramend = memblock_end_of_DRAM(); + + /* Reserve memory occupied by kernel. */ + memblock_reserve(__pa_symbol(&_text), + __pa_symbol(&_end) - __pa_symbol(&_text)); + + /* + * Reserve any memory between the start of RAM and PHYS_OFFSET + */ + if (ramstart > PHYS_OFFSET) + memblock_reserve(PHYS_OFFSET, ramstart - PHYS_OFFSET); + + if (PFN_UP(ramstart) > ARCH_PFN_OFFSET) { + pr_info("Wasting %lu bytes for tracking %lu unused pages\n", + (unsigned long)((PFN_UP(ramstart) - ARCH_PFN_OFFSET) * sizeof(struct page)), + (unsigned long)(PFN_UP(ramstart) - ARCH_PFN_OFFSET)); + } + + min_low_pfn = ARCH_PFN_OFFSET; + max_pfn = PFN_DOWN(ramend); + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { + /* + * Skip highmem here so we get an accurate max_low_pfn if low + * memory stops short of high memory. + * If the region overlaps HIGHMEM_START, end is clipped so + * max_pfn excludes the highmem portion. + */ + if (start >= PFN_DOWN(HIGHMEM_START)) + continue; + if (end > PFN_DOWN(HIGHMEM_START)) + end = PFN_DOWN(HIGHMEM_START); + if (end > max_low_pfn) + max_low_pfn = end; + } + + if (min_low_pfn >= max_low_pfn) + panic("Incorrect memory mapping !!!"); + + if (max_pfn > PFN_DOWN(HIGHMEM_START)) { +#ifdef CONFIG_HIGHMEM + highstart_pfn = PFN_DOWN(HIGHMEM_START); + highend_pfn = max_pfn; +#else + max_low_pfn = PFN_DOWN(HIGHMEM_START); + max_pfn = max_low_pfn; +#endif + } +} + +#ifdef CONFIG_ARCH_WRITECOMBINE +bool wc_enabled = true; +#else +bool wc_enabled = false; +#endif + +EXPORT_SYMBOL(wc_enabled); + +static int __init setup_writecombine(char *p) +{ + if (!strcmp(p, "on")) + wc_enabled = true; + else if (!strcmp(p, "off")) + wc_enabled = false; + else + pr_warn("Unknown writecombine setting \"%s\".\n", p); + + return 0; +} +early_param("writecombine", setup_writecombine); + +static int usermem __initdata; + +static int __init early_parse_mem(char *p) +{ + phys_addr_t start, size; + + if (!p) { + pr_err("mem parameter is empty, do nothing\n"); + return -EINVAL; + } + + /* + * If a user specifies memory size, we + * blow away any automatically generated + * size. + */ + if (usermem == 0) { + usermem = 1; + memblock_remove(memblock_start_of_DRAM(), + memblock_end_of_DRAM() - memblock_start_of_DRAM()); + } + start = 0; + size = memparse(p, &p); + if (*p == '@') + start = memparse(p + 1, &p); + else { + pr_err("Invalid format!\n"); + return -EINVAL; + } + + if (!IS_ENABLED(CONFIG_NUMA)) + memblock_add(start, size); + else + memblock_add_node(start, size, pa_to_nid(start)); + + return 0; +} +early_param("mem", early_parse_mem); + +void __init arch_reserve_vmcore(void) +{ +#ifdef CONFIG_PROC_VMCORE + u64 i; + phys_addr_t start, end; + + if (!is_kdump_kernel()) + return; + + if (!elfcorehdr_size) { + for_each_mem_range(i, &start, &end) { + if (elfcorehdr_addr >= start && elfcorehdr_addr < end) { + /* + * Reserve from the elf core header to the end of + * the memory segment, that should all be kdump + * reserved memory. + */ + elfcorehdr_size = end - elfcorehdr_addr; + break; + } + } + } + + if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) { + pr_warn("elfcorehdr is overlapped\n"); + return; + } + + memblock_reserve(elfcorehdr_addr, elfcorehdr_size); + + pr_info("Reserving %llu KiB of memory at 0x%llx for elfcorehdr\n", + elfcorehdr_size >> 10, elfcorehdr_addr); +#endif +} + +/* 2MB alignment for crash kernel regions */ +#define CRASH_ALIGN SZ_2M +#define CRASH_ADDR_MAX SZ_4G + +void __init arch_parse_crashkernel(void) +{ +#ifdef CONFIG_KEXEC + int ret; + unsigned long long total_mem; + unsigned long long crash_base, crash_size; + + total_mem = memblock_phys_mem_size(); + ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base); + if (ret < 0 || crash_size <= 0) + return; + + if (crash_base <= 0) { + crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, CRASH_ALIGN, CRASH_ADDR_MAX); + if (!crash_base) { + pr_warn("crashkernel reservation failed - No suitable area found.\n"); + return; + } + } else if (!memblock_phys_alloc_range(crash_size, CRASH_ALIGN, crash_base, crash_base + crash_size)) { + pr_warn("Invalid memory region reserved for crash kernel\n"); + return; + } + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; +#endif +} + +static void __init check_kernel_sections_mem(void) +{ + phys_addr_t start = __pa_symbol(&_text); + phys_addr_t size = __pa_symbol(&_end) - start; + + if (!memblock_is_region_memory(start, size)) { + pr_info("Kernel sections are not in the memory maps\n"); + memblock_add(start, size); + } +} + +/* + * arch_mem_init - initialize memory management subsystem + */ +static void __init arch_mem_init(char **cmdline_p) +{ + /* call board setup routine */ + plat_mem_setup(); + memblock_set_bottom_up(true); + + if (usermem) + pr_info("User-defined physical RAM map overwrite\n"); + + check_kernel_sections_mem(); + + early_init_fdt_reserve_self(); + early_init_fdt_scan_reserved_mem(); + + if (initial_boot_params) + dt_bootmem_init(); + + device_tree_init(); + + /* + * In order to reduce the possibility of kernel panic when failed to + * get IO TLB memory under CONFIG_SWIOTLB, it is better to allocate + * low memory as small as possible before swiotlb_init(), so make + * sparse_init() using top-down allocation. + */ + memblock_set_bottom_up(false); + sparse_init(); + memblock_set_bottom_up(true); + + swiotlb_init(1); + + dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); + + /* Reserve for hibernation. */ + register_nosave_region(PFN_DOWN(__pa_symbol(&__nosave_begin)), + PFN_UP(__pa_symbol(&__nosave_end))); + + memblock_dump_all(); + + early_memtest(PFN_PHYS(ARCH_PFN_OFFSET), PFN_PHYS(max_low_pfn)); +} + +static void __init resource_init(void) +{ + long i = 0; + size_t res_size; + struct resource *res; + struct memblock_region *region; + + code_resource.start = __pa_symbol(&_text); + code_resource.end = __pa_symbol(&_etext) - 1; + data_resource.start = __pa_symbol(&_etext); + data_resource.end = __pa_symbol(&_edata) - 1; + bss_resource.start = __pa_symbol(&__bss_start); + bss_resource.end = __pa_symbol(&__bss_stop) - 1; + + num_standard_resources = memblock.memory.cnt; + res_size = num_standard_resources * sizeof(*standard_resources); + standard_resources = memblock_alloc(res_size, SMP_CACHE_BYTES); + + for_each_mem_region(region) { + res = &standard_resources[i++]; + if (!memblock_is_nomap(region)) { + res->name = "System RAM"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + } else { + res->name = "Reserved"; + res->flags = IORESOURCE_MEM; + res->start = __pfn_to_phys(memblock_region_reserved_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_reserved_end_pfn(region)) - 1; + } + + request_resource(&iomem_resource, res); + + /* + * We don't know which RAM region contains kernel data, + * so we try it repeatedly and let the resource manager + * test it. + */ + request_resource(res, &code_resource); + request_resource(res, &data_resource); + request_resource(res, &bss_resource); + } + +#ifdef CONFIG_KEXEC + if (crashk_res.start < crashk_res.end) { + insert_resource(&iomem_resource, &crashk_res); + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n", + (unsigned long)((crashk_res.end - crashk_res.start + 1) >> 20), + (unsigned long)(crashk_res.start >> 20)); + } +#endif +} + +static int __init reserve_memblock_reserved_regions(void) +{ + u64 i, j; + + for (i = 0; i < num_standard_resources; ++i) { + struct resource *mem = &standard_resources[i]; + phys_addr_t r_start, r_end, mem_size = resource_size(mem); + + if (!memblock_is_region_reserved(mem->start, mem_size)) + continue; + + for_each_reserved_mem_range(j, &r_start, &r_end) { + resource_size_t start, end; + + start = max(PFN_PHYS(PFN_DOWN(r_start)), mem->start); + end = min(PFN_PHYS(PFN_UP(r_end)) - 1, mem->end); + + if (start > mem->end || end < mem->start) + continue; + + reserve_region_with_split(mem, start, end, "Reserved"); + } + } + + return 0; +} +arch_initcall(reserve_memblock_reserved_regions); + +#ifdef CONFIG_SMP +static void __init prefill_possible_map(void) +{ + int i, possible; + + possible = num_processors + disabled_cpus; + if (possible > nr_cpu_ids) + possible = nr_cpu_ids; + + pr_info("SMP: Allowing %d CPUs, %d hotplug CPUs\n", + possible, max((possible - num_processors), 0)); + + for (i = 0; i < possible; i++) + set_cpu_possible(i, true); + for (; i < NR_CPUS; i++) + set_cpu_possible(i, false); + + nr_cpu_ids = possible; +} +#else +static inline void prefill_possible_map(void) {} +#endif + +void __init setup_arch(char **cmdline_p) +{ + cpu_probe(); + *cmdline_p = boot_command_line; + unwind_init(); + + early_init(); + pagetable_init(); + parse_early_param(); + reserve_initrd_mem(); + + platform_init(); + arch_mem_init(cmdline_p); + + resource_init(); + plat_smp_setup(); + prefill_possible_map(); + + paging_init(); + +#if defined(CONFIG_KASAN) + kasan_init(); +#endif +} diff --git a/arch/loongarch/kernel/signal-common.h b/arch/loongarch/kernel/signal-common.h new file mode 100644 index 0000000000000000000000000000000000000000..a6598c3010a564e8349d020dbf6078b6e6e80cc3 --- /dev/null +++ b/arch/loongarch/kernel/signal-common.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +*/ + +#ifndef __SIGNAL_COMMON_H +#define __SIGNAL_COMMON_H + +/* #define DEBUG_SIG */ + +#ifdef DEBUG_SIG +# define DEBUGP(fmt, args...) printk("%s: " fmt, __func__, ##args) +#else +# define DEBUGP(fmt, args...) +#endif + +struct _ctx_layout { + struct sctx_info *addr; + unsigned int size; /* used at save context */ +}; + +struct extctx_layout { + unsigned long size; + unsigned int flags; + struct _ctx_layout fpu; + struct _ctx_layout lsx; + struct _ctx_layout lasx; + struct _ctx_layout lbt; + struct _ctx_layout end; +}; + +/* + * Determine which stack to use.. + */ +extern void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, + struct extctx_layout *extctx); +/* Check and clear pending FPU exceptions in saved CSR */ +extern int fcsr_pending(unsigned int __user *fcsr); + +/* Make sure we will not lose FPU ownership */ +#define lock_fpu_owner() ({ preempt_disable(); pagefault_disable(); }) +#define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); }) + +/* Assembly functions to move context to/from the FPU */ +extern asmlinkage int +_save_fp_context(void __user *fpregs, void __user *fcc, void __user *csr); +extern asmlinkage int +_restore_fp_context(void __user *fpregs, void __user *fcc, void __user *csr); +extern asmlinkage int +_save_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_restore_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_save_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +extern asmlinkage int +_restore_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr); +#if defined(CONFIG_CPU_HAS_LBT) +extern asmlinkage int +_save_scr_context(void __user *scr, void __user *eflags); +extern asmlinkage int +_restore_scr_context(void __user *scr, void __user *eflags); +#endif + +#endif /* __SIGNAL_COMMON_H */ diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c new file mode 100644 index 0000000000000000000000000000000000000000..7f758e0e3b5614c32a853a126b99ca2090df8d66 --- /dev/null +++ b/arch/loongarch/kernel/signal.c @@ -0,0 +1,985 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* Author: Hanlu Li +* Author: Huacai Chen +* Copyright (C) 2020 Loongson Technology Corporation Limited +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "signal-common.h" + +struct rt_sigframe { + struct siginfo rs_info; + struct ucontext rs_uctx; +}; + +static void __user *get_ctx_through_ctxinfo(struct sctx_info *info) +{ + return (void __user *)((char *)info + sizeof(struct sctx_info)); +} + +/* + * Thread saved context copy to/from a signal context presumed to be on the + * user stack, and therefore accessed with appropriate macros from uaccess.h. + */ +static int copy_fpu_to_sigcontext(struct fpu_context __user *ctx) +{ + int i; + int err = 0; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= + __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + ®s[i]); + } + err |= __put_user(current->thread.fpu.fcc, fcc); + err |= __put_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_fpu_from_sigcontext(struct fpu_context __user *ctx) +{ + int i; + int err = 0; + u64 fpr_val; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, ®s[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + } + err |= __get_user(current->thread.fpu.fcc, fcc); + err |= __get_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lsx_to_sigcontext(struct lsx_context __user *ctx) +{ + int i; + int err = 0; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + ®s[2*i]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 1), + ®s[2*i+1]); + } + err |= __put_user(current->thread.fpu.fcc, fcc); + err |= __put_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lsx_from_sigcontext(struct lsx_context __user *ctx) +{ + int i; + int err = 0; + u64 fpr_val; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, ®s[2*i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + err |= __get_user(fpr_val, ®s[2*i+1]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, fpr_val); + } + err |= __get_user(current->thread.fpu.fcc, fcc); + err |= __get_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lasx_to_sigcontext(struct lasx_context __user *ctx) +{ + int i; + int err = 0; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + ®s[4*i]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 1), + ®s[4*i+1]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 2), + ®s[4*i+2]); + err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 3), + ®s[4*i+3]); + } + err |= __put_user(current->thread.fpu.fcc, fcc); + err |= __put_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +static int copy_lasx_from_sigcontext(struct lasx_context __user *ctx) +{ + int i; + int err = 0; + u64 fpr_val; + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, ®s[4*i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+1]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+2]); + set_fpr64(¤t->thread.fpu.fpr[i], 2, fpr_val); + err |= __get_user(fpr_val, ®s[4*i+3]); + set_fpr64(¤t->thread.fpu.fpr[i], 3, fpr_val); + } + err |= __get_user(current->thread.fpu.fcc, fcc); + err |= __get_user(current->thread.fpu.fcsr, fcsr); + + return err; +} + +#if defined(CONFIG_CPU_HAS_LBT) +static int copy_lbt_to_sigcontext(struct lbt_context __user *ctx) +{ + int err = 0; + uint64_t __user *scr = (uint64_t *)&ctx->scr; + uint32_t __user *eflags = (uint32_t *)&ctx->eflags; + + err |= __put_user(current->thread.lbt.scr0, &scr[0]); + err |= __put_user(current->thread.lbt.scr1, &scr[1]); + err |= __put_user(current->thread.lbt.scr2, &scr[2]); + err |= __put_user(current->thread.lbt.scr3, &scr[3]); + err |= __put_user(current->thread.lbt.eflags, eflags); + + return err; +} + +static int copy_lbt_from_sigcontext(struct lbt_context __user *ctx) +{ + int err = 0; + uint64_t __user *scr = (uint64_t *)&ctx->scr; + uint32_t __user *eflags = (uint32_t *)&ctx->eflags; + + err |= __get_user(current->thread.lbt.scr0, &scr[0]); + err |= __get_user(current->thread.lbt.scr1, &scr[1]); + err |= __get_user(current->thread.lbt.scr2, &scr[2]); + err |= __get_user(current->thread.lbt.scr3, &scr[3]); + err |= __get_user(current->thread.lbt.eflags, eflags); + + return err; +} +#endif + +/* + * Wrappers for the assembly _{save,restore}_fp_context functions. + */ +static int save_hw_fpu_context(struct fpu_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _save_fp_context(regs, fcc, fcsr); +} + +static int restore_hw_fpu_context(struct fpu_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _restore_fp_context(regs, fcc, fcsr); +} + +static int save_hw_lsx_context(struct lsx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _save_lsx_context(regs, fcc, fcsr); +} + +static int restore_hw_lsx_context(struct lsx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _restore_lsx_context(regs, fcc, fcsr); +} + +static int save_hw_lasx_context(struct lasx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _save_lasx_context(regs, fcc, fcsr); +} + +static int restore_hw_lasx_context(struct lasx_context __user *ctx) +{ + uint64_t __user *regs = (uint64_t *)&ctx->regs; + uint64_t __user *fcc = &ctx->fcc; + uint32_t __user *fcsr = &ctx->fcsr; + + return _restore_lasx_context(regs, fcc, fcsr); +} + +#if defined(CONFIG_CPU_HAS_LBT) +static int save_hw_lbt_context(struct lbt_context __user *ctx) +{ + uint64_t __user *scr = (uint64_t *)&ctx->scr; + uint32_t __user *eflags = (uint32_t *)&ctx->eflags; + + return _save_scr_context(scr, eflags); +} + +static int restore_hw_lbt_context(struct lbt_context __user *ctx) +{ + uint64_t __user *scr = (uint64_t *)&ctx->scr; + uint32_t __user *eflags = (uint32_t *)&ctx->eflags; + + return _restore_scr_context(scr, eflags); +} +#endif + +/* + * Helper routines + */ +static int protected_save_fpu_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->fpu.addr; + struct fpu_context __user *fpu_ctx = (struct fpu_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&fpu_ctx->regs; + uint64_t __user *fcc = &fpu_ctx->fcc; + uint32_t __user *fcsr = &fpu_ctx->fcsr; + + while (1) { + lock_fpu_owner(); + if (is_fpu_owner()) + err = save_hw_fpu_context(fpu_ctx); + else + err = copy_fpu_to_sigcontext(fpu_ctx); + unlock_fpu_owner(); + + err |= __put_user(FPU_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->fpu.size, &info->size); + + if (likely(!err)) + break; + /* Touch the FPU context and try again */ + err = __put_user(0, ®s[0]) | + __put_user(0, ®s[31]) | + __put_user(0, fcc) | + __put_user(0, fcsr); + if (err) + return err; /* really bad sigcontext */ + } + + return err; +} + +static int protected_restore_fpu_context(struct extctx_layout *extctx) +{ + int err = 0, sig = 0, tmp __maybe_unused; + struct sctx_info __user *info = extctx->fpu.addr; + struct fpu_context __user *fpu_ctx = (struct fpu_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&fpu_ctx->regs; + uint64_t __user *fcc = &fpu_ctx->fcc; + uint32_t __user *fcsr = &fpu_ctx->fcsr; + + err = sig = fcsr_pending(fcsr); + if (err < 0) + return err; + + while (1) { + lock_fpu_owner(); + if (is_fpu_owner()) + err = restore_hw_fpu_context(fpu_ctx); + else + err = copy_fpu_from_sigcontext(fpu_ctx); + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the FPU context and try again */ + err = __get_user(tmp, ®s[0]) | + __get_user(tmp, ®s[31]) | + __get_user(tmp, fcc) | + __get_user(tmp, fcsr); + if (err) + break; /* really bad sigcontext */ + } + + return err ?: sig; +} + +static int protected_save_lsx_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->lsx.addr; + struct lsx_context __user *lsx_ctx = (struct lsx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lsx_ctx->regs; + uint64_t __user *fcc = &lsx_ctx->fcc; + uint32_t __user *fcsr = &lsx_ctx->fcsr; + + while (1) { + lock_fpu_owner(); + if (is_lsx_enabled()) + err = save_hw_lsx_context(lsx_ctx); + else { + if (is_fpu_owner()) + save_fp(current); + err = copy_lsx_to_sigcontext(lsx_ctx); + } + unlock_fpu_owner(); + + err |= __put_user(LSX_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->lsx.size, &info->size); + + if (likely(!err)) + break; + /* Touch the LSX context and try again */ + err = __put_user(0, ®s[0]) | + __put_user(0, ®s[32*2-1]) | + __put_user(0, fcc) | + __put_user(0, fcsr); + if (err) + return err; /* really bad sigcontext */ + } + + return err; +} + +static int protected_restore_lsx_context(struct extctx_layout *extctx) +{ + int err = 0, sig = 0, tmp __maybe_unused; + struct sctx_info __user *info = extctx->lsx.addr; + struct lsx_context __user *lsx_ctx = (struct lsx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lsx_ctx->regs; + uint64_t __user *fcc = &lsx_ctx->fcc; + uint32_t __user *fcsr = &lsx_ctx->fcsr; + + err = sig = fcsr_pending(fcsr); + if (err < 0) + return err; + + while (1) { + lock_fpu_owner(); + if (is_lsx_enabled()) + err = restore_hw_lsx_context(lsx_ctx); + else { + err = copy_lsx_from_sigcontext(lsx_ctx); + if (is_fpu_owner()) + restore_fp(current); + } + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the LSX context and try again */ + err = __get_user(tmp, ®s[0]) | + __get_user(tmp, ®s[32*2-1]) | + __get_user(tmp, fcc) | + __get_user(tmp, fcsr); + if (err) + break; /* really bad sigcontext */ + } + + return err ?: sig; +} + +static int protected_save_lasx_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->lasx.addr; + struct lasx_context __user *lasx_ctx = + (struct lasx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lasx_ctx->regs; + uint64_t __user *fcc = &lasx_ctx->fcc; + uint32_t __user *fcsr = &lasx_ctx->fcsr; + + while (1) { + lock_fpu_owner(); + if (is_lasx_enabled()) + err = save_hw_lasx_context(lasx_ctx); + else { + if (is_lsx_enabled()) + save_lsx(current); + else if (is_fpu_owner()) + save_fp(current); + err = copy_lasx_to_sigcontext(lasx_ctx); + } + unlock_fpu_owner(); + + err |= __put_user(LASX_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->lasx.size, &info->size); + + if (likely(!err)) + break; + /* Touch the LASX context and try again */ + err = __put_user(0, ®s[0]) | + __put_user(0, ®s[32*4-1]) | + __put_user(0, fcc) | + __put_user(0, fcsr); + if (err) + return err; /* really bad sigcontext */ + } + + return err; +} + +static int protected_restore_lasx_context(struct extctx_layout *extctx) +{ + int err = 0, sig = 0, tmp __maybe_unused; + struct sctx_info __user *info = extctx->lasx.addr; + struct lasx_context __user *lasx_ctx = + (struct lasx_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *regs = (uint64_t *)&lasx_ctx->regs; + uint64_t __user *fcc = &lasx_ctx->fcc; + uint32_t __user *fcsr = &lasx_ctx->fcsr; + + err = sig = fcsr_pending(fcsr); + if (err < 0) + return err; + + while (1) { + lock_fpu_owner(); + if (is_lasx_enabled()) + err = restore_hw_lasx_context(lasx_ctx); + else { + err = copy_lasx_from_sigcontext(lasx_ctx); + if (is_lsx_enabled()) + restore_lsx(current); + else if (is_fpu_owner()) + restore_fp(current); + } + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the LASX context and try again */ + err = __get_user(tmp, ®s[0]) | + __get_user(tmp, ®s[32*4-1]) | + __get_user(tmp, fcc) | + __get_user(tmp, fcsr); + if (err) + break; /* really bad sigcontext */ + } + + return err ?: sig; +} + +#if defined(CONFIG_CPU_HAS_LBT) +static int protected_save_lbt_context(struct extctx_layout *extctx) +{ + int err = 0; + struct sctx_info __user *info = extctx->lbt.addr; + struct lbt_context __user *lbt_ctx = (struct lbt_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *scr = (uint64_t *)&lbt_ctx->scr; + uint32_t __user *eflags = (uint32_t *)&lbt_ctx->eflags; + + while (1) { + lock_fpu_owner(); + if (is_lbt_owner()) + err = save_hw_lbt_context(lbt_ctx); + else + err = copy_lbt_to_sigcontext(lbt_ctx); + unlock_fpu_owner(); + + err |= __put_user(LBT_CTX_MAGIC, &info->magic); + err |= __put_user(extctx->lbt.size, &info->size); + + if (likely(!err)) + break; + /* Touch the LBT context and try again */ + err = __put_user(0, &scr[0]) | __put_user(0, eflags); + + if (err) + return err; + } + + return err; +} + +static int protected_restore_lbt_context(struct extctx_layout *extctx) +{ + int err = 0, tmp; + struct sctx_info __user *info = extctx->lbt.addr; + struct lbt_context __user *lbt_ctx = (struct lbt_context *)get_ctx_through_ctxinfo(info); + uint64_t __user *scr = (uint64_t *)&lbt_ctx->scr; + uint32_t __user *eflags = (uint32_t *)&lbt_ctx->eflags; + + while (1) { + lock_fpu_owner(); + if (is_lbt_owner()) + err = restore_hw_lbt_context(lbt_ctx); + else + err = copy_lbt_from_sigcontext(lbt_ctx); + unlock_fpu_owner(); + + if (likely(!err)) + break; + /* Touch the LBT context and try again */ + err = __get_user(tmp, &scr[0]) | __get_user(tmp, eflags); + + if (err) + return err; + } + + return err; +} +#endif + +static int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc, + struct extctx_layout *extctx) +{ + int i, err = 0; + struct sctx_info __user *info; + + err |= __put_user(regs->csr_era, &sc->sc_pc); + err |= __put_user(extctx->flags, &sc->sc_flags); + + err |= __put_user(0, &sc->sc_regs[0]); + for (i = 1; i < 32; i++) + err |= __put_user(regs->regs[i], &sc->sc_regs[i]); + + if (extctx->lasx.addr) + err |= protected_save_lasx_context(extctx); + else if (extctx->lsx.addr) + err |= protected_save_lsx_context(extctx); + else if (extctx->fpu.addr) + err |= protected_save_fpu_context(extctx); + +#if defined(CONFIG_CPU_HAS_LBT) + if (extctx->lbt.addr) + err |= protected_save_lbt_context(extctx); +#endif + /* Set the "end" magic */ + info = (struct sctx_info *)extctx->end.addr; + err |= __put_user(0, &info->magic); + err |= __put_user(0, &info->size); + + return err; +} + +int fcsr_pending(unsigned int __user *fcsr) +{ + int err, sig = 0; + unsigned int csr, enabled; + + err = __get_user(csr, fcsr); + enabled = ((csr & FPU_CSR_ALL_E) << 24); + /* + * If the signal handler set some FPU exceptions, clear it and + * send SIGFPE. + */ + if (csr & enabled) { + csr &= ~enabled; + err |= __put_user(csr, fcsr); + sig = SIGFPE; + } + return err ?: sig; +} + +static int parse_extcontext(struct sigcontext __user *sc, struct extctx_layout *extctx) +{ + int err = 0; + unsigned int magic, size; + struct sctx_info __user *info = (struct sctx_info __user *)&sc->sc_extcontext; + + while(1) { + err |= __get_user(magic, &info->magic); + err |= __get_user(size, &info->size); + if (err) + return err; + + switch (magic) { + case 0: /* END */ + goto done; + + case FPU_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct fpu_context))) + goto invalid; + extctx->fpu.addr = info; + break; + + case LSX_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct lsx_context))) + goto invalid; + extctx->lsx.addr = info; + break; + + case LASX_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct lasx_context))) + goto invalid; + extctx->lasx.addr = info; + break; + + case LBT_CTX_MAGIC: + if (size < (sizeof(struct sctx_info) + + sizeof(struct lbt_context))) + goto invalid; + extctx->lbt.addr = info; + break; + + default: + goto invalid; + } + + info = (struct sctx_info *)((char *)info + size); + } + +done: + return 0; + +invalid: + return -EINVAL; +} + +static int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) +{ + int i, err = 0; + struct extctx_layout extctx; + + memset(&extctx, 0, sizeof(struct extctx_layout)); + + err = __get_user(extctx.flags, &sc->sc_flags); + if (err) + goto bad; + + err = parse_extcontext(sc, &extctx); + if (err) + goto bad; + + conditional_used_math(extctx.flags & SC_USED_FP); + + /* + * The signal handler may have used FPU; give it up if the program + * doesn't want it following sigreturn. + */ + if (!(extctx.flags & SC_USED_FP)) + lose_fpu(0); + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + err |= __get_user(regs->csr_era, &sc->sc_pc); + for (i = 1; i < 32; i++) + err |= __get_user(regs->regs[i], &sc->sc_regs[i]); + + if (extctx.lasx.addr) + err |= protected_restore_lasx_context(&extctx); + else if (extctx.lsx.addr) + err |= protected_restore_lsx_context(&extctx); + else if (extctx.fpu.addr) + err |= protected_restore_fpu_context(&extctx); + +#if defined(CONFIG_CPU_HAS_LBT) + if (extctx.lbt.addr) + err |= protected_restore_lbt_context(&extctx); +#endif + +bad: + return err; +} + +static unsigned int handle_flags(void) +{ + unsigned int flags = 0; + + flags = used_math() ? SC_USED_FP : 0; + + switch (current->thread.error_code) { + case 1: + flags |= SC_ADDRERR_RD; + break; + case 2: + flags |= SC_ADDRERR_WR; + break; + } + + return flags; +} + +static unsigned long extframe_alloc(struct extctx_layout *extctx, + struct _ctx_layout *layout, + size_t size, unsigned int align, unsigned long base) +{ + unsigned long new_base = base - size; + + new_base = round_down(new_base, (align < 16 ? 16 : align)); + new_base -= sizeof(struct sctx_info); + + layout->addr = (void *)new_base; + layout->size = (unsigned int)(base - new_base); + extctx->size += layout->size; + + return new_base; +} + +static unsigned long setup_extcontext(struct extctx_layout *extctx, unsigned long sp) +{ + unsigned long new_sp = sp; + + memset(extctx, 0, sizeof(struct extctx_layout)); + + extctx->flags = handle_flags(); + + /* Grow down, alloc "end" context info first. */ + new_sp -= sizeof(struct sctx_info); + extctx->end.addr = (void *)new_sp; + extctx->end.size = (unsigned int)sizeof(struct sctx_info); + extctx->size += extctx->end.size; + + if (extctx->flags & SC_USED_FP) { + if (cpu_has_lasx && thread_lasx_context_live()) + new_sp = extframe_alloc(extctx, &extctx->lasx, + sizeof(struct lasx_context), LASX_CTX_ALIGN, new_sp); + else if (cpu_has_lsx && thread_lsx_context_live()) + new_sp = extframe_alloc(extctx, &extctx->lsx, + sizeof(struct lsx_context), LSX_CTX_ALIGN, new_sp); + else if (cpu_has_fpu) + new_sp = extframe_alloc(extctx, &extctx->fpu, + sizeof(struct fpu_context), FPU_CTX_ALIGN, new_sp); + } + +#if defined(CONFIG_CPU_HAS_LBT) + if (cpu_has_lbt && thread_lbt_context_live()) + new_sp = extframe_alloc(extctx, &extctx->lbt, + sizeof(struct lbt_context), LBT_CTX_ALIGN, new_sp); +#endif + + return new_sp; +} + +void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, + struct extctx_layout *extctx) +{ + unsigned long sp; + + /* Default to using normal stack */ + sp = regs->regs[3]; + + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && + !likely(on_sig_stack(sp - sizeof(struct rt_sigframe)))) + return (void __user __force *)(-1UL); + + sp = sigsp(sp, ksig); + sp = round_down(sp, 16); + sp = setup_extcontext(extctx, sp); + sp -= sizeof(struct rt_sigframe); + + if (!IS_ALIGNED(sp, 16)) + BUG(); + + return (void __user *)sp; +} + +/* + * Atomically swap in the new signal mask, and wait for a signal. + */ + +SYSCALL_DEFINE0(rt_sigreturn) +{ + int sig; + sigset_t set; + struct pt_regs *regs; + struct rt_sigframe __user *frame; + + regs = current_pt_regs(); + frame = (struct rt_sigframe __user *)regs->regs[3]; + if (!access_ok(frame, sizeof(*frame))) + goto badframe; + if (__copy_from_user(&set, &frame->rs_uctx.uc_sigmask, sizeof(set))) + goto badframe; + + set_current_blocked(&set); + + sig = restore_sigcontext(regs, &frame->rs_uctx.uc_mcontext); + if (sig < 0) + goto badframe; + else if (sig) + force_sig(sig); + + regs->regs[0] = 0; /* No syscall restarting */ + if (restore_altstack(&frame->rs_uctx.uc_stack)) + goto badframe; + + return regs->regs[4]; + +badframe: + force_sig(SIGSEGV); + return 0; +} + +static int setup_rt_frame(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) +{ + int err = 0; + struct extctx_layout extctx; + struct rt_sigframe __user *frame; + + frame = get_sigframe(ksig, regs, &extctx); + if (!access_ok(frame, sizeof(*frame) + extctx.size)) + return -EFAULT; + + /* Create siginfo. */ + err |= copy_siginfo_to_user(&frame->rs_info, &ksig->info); + + /* Create the ucontext. */ + err |= __put_user(0, &frame->rs_uctx.uc_flags); + err |= __put_user(NULL, &frame->rs_uctx.uc_link); + err |= __save_altstack(&frame->rs_uctx.uc_stack, regs->regs[3]); + err |= setup_sigcontext(regs, &frame->rs_uctx.uc_mcontext, &extctx); + err |= __copy_to_user(&frame->rs_uctx.uc_sigmask, set, sizeof(*set)); + + if (err) + return -EFAULT; + + /* + * Arguments to signal handler: + * + * a0 = signal number + * a1 = pointer to siginfo + * a2 = pointer to ucontext + * + * c0_era point to the signal handler, $r3 (sp) points to + * the struct rt_sigframe. + */ +#ifdef CONFIG_CPU_HAS_LBT + if (current_thread_info()->tp_value) + regs->regs[2] = current_thread_info()->tp_value; +#endif + regs->regs[4] = ksig->sig; + regs->regs[5] = (unsigned long) &frame->rs_info; + regs->regs[6] = (unsigned long) &frame->rs_uctx; + regs->regs[3] = (unsigned long) frame; + regs->regs[1] = (unsigned long) sig_return; + regs->csr_era = (unsigned long) ksig->ka.sa.sa_handler; + + DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", + current->comm, current->pid, + frame, regs->csr_era, regs->regs[1]); + + return 0; +} + +static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) +{ + int ret; + sigset_t *oldset = sigmask_to_save(); + void *vdso = current->mm->context.vdso; + + /* Are we from a system call? */ + if (regs->regs[0]) { + switch(regs->regs[4]) { + case -ERESTART_RESTARTBLOCK: + case -ERESTARTNOHAND: + regs->regs[4] = -EINTR; + break; + case -ERESTARTSYS: + if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { + regs->regs[4] = -EINTR; + break; + } + fallthrough; + case -ERESTARTNOINTR: + regs->regs[4] = regs->orig_a0; + regs->csr_era -= 4; + } + + regs->regs[0] = 0; /* Don't deal with this again. */ + } + + rseq_signal_deliver(ksig, regs); + + ret = setup_rt_frame(vdso + current->thread.vdso->offset_sigreturn, ksig, regs, oldset); + + signal_setup_done(ret, ksig, 0); +} + +void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) +{ + struct ksignal ksig; + + if (has_signal && get_signal(&ksig)) { + /* Whee! Actually deliver the signal. */ + handle_signal(&ksig, regs); + return; + } + + /* Are we from a system call? */ + if (regs->regs[0]) { + switch (regs->regs[4]) { + case -ERESTARTNOHAND: + case -ERESTARTSYS: + case -ERESTARTNOINTR: + regs->regs[4] = regs->orig_a0; + regs->csr_era -= 4; + break; + + case -ERESTART_RESTARTBLOCK: + regs->regs[4] = regs->orig_a0; + regs->regs[11] = __NR_restart_syscall; + regs->csr_era -= 4; + break; + } + regs->regs[0] = 0; /* Don't deal with this again. */ + } + + /* + * If there's no signal to deliver, we just put the saved sigmask + * back + */ + restore_saved_sigmask(); +} diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c new file mode 100644 index 0000000000000000000000000000000000000000..695d0b437faefee431a917497c4a99f082c9c651 --- /dev/null +++ b/arch/loongarch/kernel/smp.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ +EXPORT_SYMBOL(__cpu_number_map); + +int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ +EXPORT_SYMBOL(__cpu_logical_map); + +/* Representing the threads (siblings) of each logical CPU */ +cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(cpu_sibling_map); + +/* Representing the core map of multi-core chips of each logical CPU */ +cpumask_t cpu_core_map[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(cpu_core_map); + +static DECLARE_COMPLETION(cpu_starting); +static DECLARE_COMPLETION(cpu_running); + +/* + * A logcal cpu mask containing only one VPE per core to + * reduce the number of IPIs on large MT systems. + */ +cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(cpu_foreign_map); + +/* representing cpus for which sibling maps can be computed */ +static cpumask_t cpu_sibling_setup_map; + +/* representing cpus for which core maps can be computed */ +static cpumask_t cpu_core_setup_map; + +static inline void set_cpu_core_map(int cpu) +{ + int i; + + cpumask_set_cpu(cpu, &cpu_core_setup_map); + + for_each_cpu(i, &cpu_core_setup_map) { + if (cpu_data[cpu].package == cpu_data[i].package) { + cpumask_set_cpu(i, &cpu_core_map[cpu]); + cpumask_set_cpu(cpu, &cpu_core_map[i]); + } + } +} + +void set_cpu_sibling_map(int cpu) +{ + int i; + + cpumask_set_cpu(cpu, &cpu_sibling_setup_map); + + for_each_cpu(i, &cpu_sibling_setup_map) { + if (cpus_are_siblings(cpu, i)) { + cpumask_set_cpu(i, &cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[i]); + } + } +} + +void clear_cpu_sibling_map(int cpu) +{ + int i; + + for_each_cpu(i, &cpu_sibling_setup_map) { + if (cpus_are_siblings(cpu, i)) { + cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); + cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); + } + } + + cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); +} + +/* + * Calculate a new cpu_foreign_map mask whenever a + * new cpu appears or disappears. + */ +void calculate_cpu_foreign_map(void) +{ + int i, k, core_present; + cpumask_t temp_foreign_map; + + /* Re-calculate the mask */ + cpumask_clear(&temp_foreign_map); + for_each_online_cpu(i) { + core_present = 0; + for_each_cpu(k, &temp_foreign_map) + if (cpus_are_siblings(i, k)) + core_present = 1; + if (!core_present) + cpumask_set_cpu(i, &temp_foreign_map); + } + + for_each_online_cpu(i) + cpumask_andnot(&cpu_foreign_map[i], + &temp_foreign_map, &cpu_sibling_map[i]); +} + +struct plat_smp_ops *mp_ops; +EXPORT_SYMBOL(mp_ops); + +void register_smp_ops(const struct plat_smp_ops *ops) +{ + if (mp_ops) + printk(KERN_WARNING "Overriding previously set SMP ops\n"); + + mp_ops = (struct plat_smp_ops *)ops; + pv_ipi_init(); +} + +/* + * First C code run on the secondary CPUs after being started up by + * the master. + */ +asmlinkage void start_secondary(void) +{ + unsigned int cpu; + + sync_counter(); + cpu = raw_smp_processor_id(); + set_my_cpu_offset(per_cpu_offset(cpu)); + + cpu_probe(); + constant_clockevent_init(); + mp_ops->init_secondary(); + + set_cpu_sibling_map(cpu); + set_cpu_core_map(cpu); + + notify_cpu_starting(cpu); + + /* Notify boot CPU that we're starting */ + complete(&cpu_starting); + + /* The CPU is running, now mark it online */ + set_cpu_online(cpu, true); + + calculate_cpu_foreign_map(); + + /* + * Notify boot CPU that we're up & online and it can safely return + * from __cpu_up + */ + complete(&cpu_running); + + /* + * irq will be enabled in ->smp_finish(), enabling it too early + * is dangerous. + */ + WARN_ON_ONCE(!irqs_disabled()); + mp_ops->smp_finish(); + + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); +} + +static void stop_this_cpu(void *dummy) +{ + /* + * Remove this CPU: + */ + + set_cpu_online(smp_processor_id(), false); + calculate_cpu_foreign_map(); + local_irq_disable(); + while (1); +} + +void smp_send_stop(void) +{ + smp_call_function(stop_this_cpu, NULL, 0); +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ +} + +/* called from main before smp_init() */ +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + init_new_context(current, &init_mm); + current_thread_info()->cpu = 0; + mp_ops->prepare_cpus(max_cpus); + set_cpu_sibling_map(0); + set_cpu_core_map(0); + calculate_cpu_foreign_map(); +#ifndef CONFIG_HOTPLUG_CPU + init_cpu_present(cpu_possible_mask); +#endif +} + +static void __init kvm_smp_prepare_boot_cpu(void) +{ + kvm_spinlock_init(); +} + +/* Preload SMP state for boot cpu */ +void smp_prepare_boot_cpu(void) +{ + unsigned int cpu, node, rr_node; + + set_cpu_possible(0, true); + set_cpu_online(0, true); + set_my_cpu_offset(per_cpu_offset(0)); + + rr_node = first_node(node_online_map); + for_each_possible_cpu(cpu) { + node = early_cpu_to_node(cpu); + + /* + * The mapping between present cpus and nodes has been + * built during MADT and SRAT parsing. + * + * If possible cpus = present cpus here, early_cpu_to_node + * will return valid node. + * + * If possible cpus > present cpus here (e.g. some possible + * cpus will be added by cpu-hotplug later), for possible but + * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, + * and we just map them to online nodes in round-robin way. + * Once hotplugged, new correct mapping will be built for them. + * */ + if (node != NUMA_NO_NODE) + set_cpu_numa_node(cpu, node); + else { + set_cpu_numa_node(cpu, rr_node); + rr_node = next_node_in(rr_node, node_online_map); + } + } + + kvm_smp_prepare_boot_cpu(); +} + +int __cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + int err; + + err = mp_ops->boot_secondary(cpu, tidle); + if (err) + return err; + + /* Wait for CPU to start and be ready to sync counters */ + if (!wait_for_completion_timeout(&cpu_starting, + msecs_to_jiffies(5000))) { + pr_crit("CPU%u: failed to start\n", cpu); + return -EIO; + } + + /* Wait for CPU to finish startup & mark itself online before return */ + wait_for_completion(&cpu_running); + return 0; +} + +#ifdef CONFIG_PROFILING +/* Not really SMP stuff ... */ +int setup_profiling_timer(unsigned int multiplier) +{ + return 0; +} +#endif + +static void flush_tlb_all_ipi(void *info) +{ + local_flush_tlb_all(); +} + +void flush_tlb_all(void) +{ + on_each_cpu(flush_tlb_all_ipi, NULL, 1); +} + +static void flush_tlb_mm_ipi(void *mm) +{ + local_flush_tlb_mm((struct mm_struct *)mm); +} + +/* + * Special Variant of smp_call_function for use by TLB functions: + * + * o No return value + * o collapses to normal function call on UP kernels + * o collapses to normal function call on systems with a single shared + * primary cache. + */ +static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) +{ + smp_call_function(func, info, 1); +} + +static inline void smp_on_each_tlb(void (*func) (void *info), void *info) +{ + preempt_disable(); + + smp_on_other_tlbs(func, info); + func(info); + + preempt_enable(); +} + +/* + * The following tlb flush calls are invoked when old translations are + * being torn down, or pte attributes are changing. For single threaded + * address spaces, a new context is obtained on the current cpu, and tlb + * context on other cpus are invalidated to force a new context allocation + * at switch_mm time, should the mm ever be used on other cpus. For + * multithreaded address spaces, intercpu interrupts have to be sent. + * Another case where intercpu interrupts are required is when the target + * mm might be active on another cpu (eg debuggers doing the flushes on + * behalf of debugees, kswapd stealing pages from another process etc). + */ + +void flush_tlb_mm(struct mm_struct *mm) +{ + if (atomic_read(&mm->mm_users) == 0) + return; /* happens as a result of exit_mmap() */ + + preempt_disable(); + + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); + } else { + unsigned int cpu; + + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) + cpu_context(cpu, mm) = 0; + } + local_flush_tlb_mm(mm); + } + + preempt_enable(); +} + +struct flush_tlb_data { + struct vm_area_struct *vma; + unsigned long addr1; + unsigned long addr2; +}; + +static void flush_tlb_range_ipi(void *info) +{ + struct flush_tlb_data *fd = info; + + local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); +} + +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + preempt_disable(); + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + struct flush_tlb_data fd = { + .vma = vma, + .addr1 = start, + .addr2 = end, + }; + + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); + } else { + unsigned int cpu; + + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) + cpu_context(cpu, mm) = 0; + } + local_flush_tlb_range(vma, start, end); + } + preempt_enable(); +} + +static void flush_tlb_kernel_range_ipi(void *info) +{ + struct flush_tlb_data *fd = info; + + local_flush_tlb_kernel_range(fd->addr1, fd->addr2); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + struct flush_tlb_data fd = { + .addr1 = start, + .addr2 = end, + }; + + on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); +} + +static void flush_tlb_page_ipi(void *info) +{ + struct flush_tlb_data *fd = info; + + local_flush_tlb_page(fd->vma, fd->addr1); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + preempt_disable(); + if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { + struct flush_tlb_data fd = { + .vma = vma, + .addr1 = page, + }; + + on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); + } else { + unsigned int cpu; + + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) + cpu_context(cpu, vma->vm_mm) = 0; + } + local_flush_tlb_page(vma, page); + } + preempt_enable(); +} + +static void flush_tlb_one_ipi(void *info) +{ + unsigned long vaddr = (unsigned long) info; + + local_flush_tlb_one(vaddr); +} + +void flush_tlb_one(unsigned long vaddr) +{ + smp_on_each_tlb(flush_tlb_one_ipi, (void *) vaddr); +} + +EXPORT_SYMBOL(flush_tlb_page); +EXPORT_SYMBOL(flush_tlb_one); diff --git a/arch/loongarch/kernel/spinlock_test.c b/arch/loongarch/kernel/spinlock_test.c new file mode 100644 index 0000000000000000000000000000000000000000..ea8462e8cb0b33dd926ef9401f655fafa6134739 --- /dev/null +++ b/arch/loongarch/kernel/spinlock_test.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +static int ss_get(void) +{ + int cont; + int loops; + u64 timeval; + ktime_t start, finish; + DEFINE_RAW_SPINLOCK(ss_spin); + + cont = 1; + loops = 1000000; + + start = ktime_get(); + + while (cont) { + raw_spin_lock(&ss_spin); + loops--; + if (loops == 0) + cont = 0; + raw_spin_unlock(&ss_spin); + } + + finish = ktime_get(); + + timeval = ktime_us_delta(finish, start); + printk("Single: %llu\n",timeval); + + return 0; +} + +struct spin_multi_state { + raw_spinlock_t lock; + atomic_t start_wait; + atomic_t enter_wait; + atomic_t exit_wait; + int loops; +}; + +struct spin_multi_per_thread { + struct spin_multi_state *state; + ktime_t start; +}; + +static int multi_other(void *data) +{ + int loops; + int cont; + struct spin_multi_per_thread *pt = data; + struct spin_multi_state *s = pt->state; + + loops = s->loops; + cont = 1; + + atomic_dec(&s->enter_wait); + + while (atomic_read(&s->enter_wait)) + ; /* spin */ + + pt->start = ktime_get(); + + atomic_dec(&s->start_wait); + + while (atomic_read(&s->start_wait)) + ; /* spin */ + + while (cont) { + raw_spin_lock(&s->lock); + loops--; + if (loops == 0) + cont = 0; + raw_spin_unlock(&s->lock); + } + + atomic_dec(&s->exit_wait); + while (atomic_read(&s->exit_wait)) + ; /* spin */ + return 0; +} + +static int multi_get(void) +{ + u64 timeval; + ktime_t finish; + struct spin_multi_state ms; + struct spin_multi_per_thread t1, t2; + + ms.lock = __RAW_SPIN_LOCK_UNLOCKED("multi_get"); + ms.loops = 1000000; + + atomic_set(&ms.start_wait, 2); + atomic_set(&ms.enter_wait, 2); + atomic_set(&ms.exit_wait, 2); + t1.state = &ms; + t2.state = &ms; + + kthread_run(multi_other, &t2, "multi_get"); + + multi_other(&t1); + + finish = ktime_get(); + + timeval = ktime_us_delta(finish, t1.start); + printk("Multiple: %llu\n",timeval); + + return 0; +} + +static int __init spinlock_test(void) +{ + ss_get(); + multi_get(); + + return 0; +} +module_init(spinlock_test); + +MODULE_LICENSE("GPL"); diff --git a/arch/loongarch/kernel/stacktrace.c b/arch/loongarch/kernel/stacktrace.c new file mode 100644 index 0000000000000000000000000000000000000000..973c39256382d28046ddbbe2ac691dc1944dc336 --- /dev/null +++ b/arch/loongarch/kernel/stacktrace.c @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Stack trace management functions + * + * Copyright (C) 2006 Atsushi Nemoto + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include + +#include +#include + +typedef bool (*stack_trace_consume_fn)(struct stack_trace *trace, + unsigned long addr); + +static bool consume_entry(struct stack_trace *trace, unsigned long addr) +{ + if (trace->nr_entries >= trace->max_entries) + return false; + + if (trace->skip > 0) { + trace->skip--; + return true; + } + + trace->entries[trace->nr_entries++] = addr; + return trace->nr_entries < trace->max_entries; +} + +static bool consume_entry_nosched(struct stack_trace *trace, + unsigned long addr) +{ + if (in_sched_functions(addr)) + return true; + return consume_entry(trace, addr); +} + +static void save_context_stack(struct task_struct *tsk, + struct stack_trace *trace, + struct pt_regs *regs, + stack_trace_consume_fn fn) +{ + struct pt_regs dummyregs; + struct unwind_state state; + unsigned long addr; + + regs = &dummyregs; + + if (tsk == current) { + regs->csr_era = (unsigned long)__builtin_return_address(0); + regs->regs[3] = (unsigned long)__builtin_frame_address(0); + } else { + regs->csr_era = thread_saved_ra(tsk); + regs->regs[3] = thread_saved_fp(tsk); + } + + regs->regs[1] = 0; + regs->regs[22] = 0; + + for (unwind_start(&state, tsk, regs); + !unwind_done(&state); unwind_next_frame(&state)) { + addr = unwind_get_return_address(&state); + if (!addr || !fn(trace, addr)) + return; + } +} + +/* + * Save stack-backtrace addresses into a stack_trace buffer. + */ +void save_stack_trace(struct stack_trace *trace) +{ + stack_trace_consume_fn consume = consume_entry; + + WARN_ON(trace->nr_entries || !trace->max_entries); + + save_context_stack(current, trace, NULL, consume); +} +EXPORT_SYMBOL_GPL(save_stack_trace); + +void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) +{ + stack_trace_consume_fn consume = consume_entry; + + /* We don't want this function nor the caller */ + trace->skip += 7; + WARN_ON(trace->nr_entries || !trace->max_entries); + + save_context_stack(current, trace, regs, consume); +} +EXPORT_SYMBOL_GPL(save_stack_trace_regs); + +void save_stack_trace_tsk(struct task_struct *tsk, + struct stack_trace *trace) +{ + stack_trace_consume_fn consume = consume_entry_nosched; + + WARN_ON(trace->nr_entries || !trace->max_entries); + + save_context_stack(tsk, trace, NULL, consume); +} +EXPORT_SYMBOL_GPL(save_stack_trace_tsk); + +#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE + +static int __always_inline +__save_stack_trace_reliable(struct stack_trace *trace, + struct task_struct *tsk) +{ + struct unwind_state state; + struct pt_regs dummyregs; + struct pt_regs *regs = &dummyregs; + unsigned long addr; + + if (tsk == current) { + regs->csr_era = (unsigned long)__builtin_return_address(0); + regs->regs[3] = (unsigned long)__builtin_frame_address(0); + } else { + regs->csr_era = thread_saved_ra(tsk); + regs->regs[3] = thread_saved_fp(tsk); + } + + for (unwind_start(&state, tsk, regs); + !unwind_done(&state) && !unwind_error(&state); + unwind_next_frame(&state)) { + + addr = unwind_get_return_address(&state); + + /* + * A NULL or invalid return address probably means there's some + * generated code which __kernel_text_address() doesn't know + * about. + */ + if (!addr) + return -EINVAL; + + if (!consume_entry(trace, addr)) + return -EINVAL; + } + + /* Check for stack corruption */ + if (unwind_error(&state)) + return -EINVAL; + + return 0; +} + +/* + * This function returns an error if it detects any unreliable features of the + * stack. Otherwise it guarantees that the stack trace is reliable. + * + * If the task is not 'current', the caller *must* ensure the task is inactive. + */ +int save_stack_trace_tsk_reliable(struct task_struct *tsk, + struct stack_trace *trace) +{ + int ret; + + /* + * If the task doesn't have a stack (e.g., a zombie), the stack is + * "reliably" empty. + */ + if (!try_get_task_stack(tsk)) + return 0; + + ret = __save_stack_trace_reliable(trace, tsk); + + put_task_stack(tsk); + + return ret; +} +#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */ + +static int +copy_stack_frame(unsigned long fp, struct stack_frame *frame) +{ + int ret; + unsigned long err; + unsigned long __user *user_frame_tail; + + user_frame_tail = (unsigned long *)(fp - sizeof(struct stack_frame)); + if (!access_ok(user_frame_tail, sizeof(*frame))) + return 0; + + ret = 1; + pagefault_disable(); + err = (__copy_from_user_inatomic(frame, user_frame_tail, sizeof(*frame))); + if (err || (unsigned long)user_frame_tail >= frame->fp) + ret = 0; + pagefault_enable(); + + return ret; +} + +static inline void __save_stack_trace_user(struct stack_trace *trace) +{ + const struct pt_regs *regs = task_pt_regs(current); + unsigned long fp = regs->regs[22]; + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = regs->csr_era; + + while (trace->nr_entries < trace->max_entries && fp && !((unsigned long)fp & 0xf)) { + struct stack_frame frame; + + frame.fp = 0; + frame.ra = 0; + if (!copy_stack_frame(fp, &frame)) + break; + if (!frame.ra) + break; + trace->entries[trace->nr_entries++] = + frame.ra; + fp = frame.fp; + } +} + +void save_stack_trace_user(struct stack_trace *trace) +{ + /* + * Trace user stack if we are not a kernel thread + */ + if (current->mm) + __save_stack_trace_user(trace); + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} diff --git a/arch/loongarch/kernel/switch.S b/arch/loongarch/kernel/switch.S new file mode 100644 index 0000000000000000000000000000000000000000..8741f6e62899e0cce2e0b36883f4a8bef97bc3f0 --- /dev/null +++ b/arch/loongarch/kernel/switch.S @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include + +#include + +/* + * task_struct *__switch_to(task_struct *prev, task_struct *next, + * struct thread_info *next_ti, + * void *sched_ra, void *sched_cfa) + */ + .align 5 +SYM_FUNC_START(__switch_to) + csrrd t1, LOONGARCH_CSR_PRMD + stptr.d t1, a0, THREAD_CSRPRMD + + cpu_save_nonscratch a0 + stptr.d ra, a0, THREAD_REG01 + stptr.d a3, a0, THREAD_SCHED_RA + stptr.d a4, a0, THREAD_SCHED_CFA +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) + la t7, __stack_chk_guard + LONG_L t8, a1, TASK_STACK_CANARY + LONG_S t8, t7, 0 +#endif + move tp, a2 + cpu_restore_nonscratch a1 + + li.w t0, _THREAD_SIZE + PTR_ADD t0, t0, tp + set_saved_sp t0, t1, t2 + + ldptr.d t1, a1, THREAD_CSRPRMD + csrwr t1, LOONGARCH_CSR_PRMD + + jr ra +SYM_FUNC_END(__switch_to) diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c new file mode 100644 index 0000000000000000000000000000000000000000..58454baa1d5935d72d9a6da03833f5a31d8e6854 --- /dev/null +++ b/arch/loongarch/kernel/syscall.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* Copyright (C) 2020 Loongson Technology Corporation Limited +* +* Author: Hanlu Li +* Author: Huacai Chen +*/ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#undef __SYSCALL +#define __SYSCALL(nr, call) [nr] = (call), + +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, + prot, unsigned long, flags, unsigned long, fd, off_t, offset) +{ + if (offset & ~PAGE_MASK) + return -EINVAL; + + return ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> PAGE_SHIFT); +} + +void *sys_call_table[__NR_syscalls] = { + [0 ... __NR_syscalls - 1] = sys_ni_syscall, +#include +}; + +typedef long (*sys_call_fn)(unsigned long, unsigned long, + unsigned long, unsigned long, unsigned long, unsigned long); + +void noinstr do_syscall(struct pt_regs *regs) +{ + unsigned long nr; + sys_call_fn syscall_fn; + + nr = regs->regs[11]; + /* Set for syscall restarting */ + if (nr < NR_syscalls) + regs->regs[0] = nr + 1; + + regs->csr_era += 4; + regs->orig_a0 = regs->regs[4]; + regs->regs[4] = -ENOSYS; + + nr = syscall_enter_from_user_mode(regs, nr); + + if (nr < NR_syscalls) { + syscall_fn = sys_call_table[nr]; + regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6], + regs->regs[7], regs->regs[8], regs->regs[9]); + } + + syscall_exit_to_user_mode(regs); +} diff --git a/arch/loongarch/kernel/sysrq.c b/arch/loongarch/kernel/sysrq.c new file mode 100644 index 0000000000000000000000000000000000000000..1f449b1610cafccc1716bb45e883bc3d424a2136 --- /dev/null +++ b/arch/loongarch/kernel/sysrq.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LoongArch specific sysrq operations. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + +#include +#include + +/* + * Dump TLB entries on all CPUs. + */ + +static DEFINE_SPINLOCK(show_lock); + +static void sysrq_tlbdump_single(void *dummy) +{ + unsigned long flags; + + spin_lock_irqsave(&show_lock, flags); + + pr_info("CPU%d:\n", smp_processor_id()); + dump_tlb_regs(); + pr_info("\n"); + dump_tlb_all(); + pr_info("\n"); + + spin_unlock_irqrestore(&show_lock, flags); +} + +#ifdef CONFIG_SMP +static void sysrq_tlbdump_othercpus(struct work_struct *dummy) +{ + smp_call_function(sysrq_tlbdump_single, NULL, 0); +} + +static DECLARE_WORK(sysrq_tlbdump, sysrq_tlbdump_othercpus); +#endif + +static void sysrq_handle_tlbdump(int key) +{ + sysrq_tlbdump_single(NULL); +#ifdef CONFIG_SMP + schedule_work(&sysrq_tlbdump); +#endif +} + +static struct sysrq_key_op sysrq_tlbdump_op = { + .handler = sysrq_handle_tlbdump, + .help_msg = "show-tlbs(x)", + .action_msg = "Show TLB entries", + .enable_mask = SYSRQ_ENABLE_DUMP, +}; + +static int __init loongarch_sysrq_init(void) +{ + return register_sysrq_key('x', &sysrq_tlbdump_op); +} +arch_initcall(loongarch_sysrq_init); diff --git a/arch/loongarch/kernel/time.c b/arch/loongarch/kernel/time.c new file mode 100644 index 0000000000000000000000000000000000000000..bcb6957a81036950e49be7e65de052909943cc3f --- /dev/null +++ b/arch/loongarch/kernel/time.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * Common time service routines for LoongArch machines. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +u64 cpu_clock_freq; +EXPORT_SYMBOL(cpu_clock_freq); +u64 const_clock_freq; +EXPORT_SYMBOL(const_clock_freq); + +static DEFINE_RAW_SPINLOCK(state_lock); +static DEFINE_PER_CPU(struct clock_event_device, constant_clockevent_device); + +static void constant_event_handler(struct clock_event_device *dev) +{ +} + +static irqreturn_t constant_timer_interrupt(int irq, void *data) +{ + int cpu = smp_processor_id(); + struct clock_event_device *cd; + + /* Clear Timer Interrupt */ + write_csr_tintclear(CSR_TINTCLR_TI); + cd = &per_cpu(constant_clockevent_device, cpu); + cd->event_handler(cd); + + return IRQ_HANDLED; +} + +static int constant_set_state_oneshot(struct clock_event_device *evt) +{ + unsigned long timer_config; + + raw_spin_lock(&state_lock); + + timer_config = csr_read64(LOONGARCH_CSR_TCFG); + timer_config |= CSR_TCFG_EN; + timer_config &= ~CSR_TCFG_PERIOD; + csr_write64(timer_config, LOONGARCH_CSR_TCFG); + + raw_spin_unlock(&state_lock); + + return 0; +} + +static int constant_set_state_periodic(struct clock_event_device *evt) +{ + unsigned long period; + unsigned long timer_config; + + raw_spin_lock(&state_lock); + + period = const_clock_freq / HZ; + timer_config = period & CSR_TCFG_VAL; + timer_config |= (CSR_TCFG_PERIOD | CSR_TCFG_EN); + csr_write64(timer_config, LOONGARCH_CSR_TCFG); + + raw_spin_unlock(&state_lock); + + return 0; +} + +static int constant_set_state_shutdown(struct clock_event_device *evt) +{ + unsigned long timer_config; + + raw_spin_lock(&state_lock); + + timer_config = csr_read64(LOONGARCH_CSR_TCFG); + timer_config &= ~CSR_TCFG_EN; + csr_write64(timer_config, LOONGARCH_CSR_TCFG); + + raw_spin_unlock(&state_lock); + + return 0; +} + +static int constant_timer_next_event(unsigned long delta, struct clock_event_device *evt) +{ + unsigned long timer_config; + + delta &= CSR_TCFG_VAL; + timer_config = delta | CSR_TCFG_EN; + csr_write64(timer_config, LOONGARCH_CSR_TCFG); + + return 0; +} + +static unsigned long __init get_loops_per_jiffy(void) +{ + unsigned long lpj = (unsigned long)const_clock_freq; + + do_div(lpj, HZ); + + return lpj; +} + +static long init_offset __nosavedata; + +void save_counter(void) +{ + init_offset = drdtime(); +} + +void sync_counter(void) +{ + /* Ensure counter begin at 0 */ + csr_write64(init_offset, LOONGARCH_CSR_CNTC); +} + +int constant_clockevent_init(void) +{ + int irq; + unsigned int cpu = smp_processor_id(); + unsigned long min_delta = 0x600; + unsigned long max_delta = (1UL << 48) - 1; + struct clock_event_device *cd; + static int timer_irq_installed = 0; + + irq = get_timer_irq(); + + cd = &per_cpu(constant_clockevent_device, cpu); + + cd->name = "Constant"; + cd->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_PERCPU; + + cd->irq = irq; + cd->rating = 320; + cd->cpumask = cpumask_of(cpu); + cd->set_state_oneshot = constant_set_state_oneshot; + cd->set_state_oneshot_stopped = constant_set_state_shutdown; + cd->set_state_periodic = constant_set_state_periodic; + cd->set_state_shutdown = constant_set_state_shutdown; + cd->set_next_event = constant_timer_next_event; + cd->event_handler = constant_event_handler; + + clockevents_config_and_register(cd, const_clock_freq, min_delta, max_delta); + + if (timer_irq_installed) + return 0; + + timer_irq_installed = 1; + + sync_counter(); + + if (request_irq(irq, constant_timer_interrupt, IRQF_PERCPU | IRQF_TIMER, "timer", NULL)) + pr_err("Failed to request irq %d (timer)\n", irq); + + lpj_fine = get_loops_per_jiffy(); + pr_info("Constant clock event device register\n"); + + return 0; +} + +static u64 read_const_counter(struct clocksource *clk) +{ + return drdtime(); +} + +static u64 native_sched_clock(void) +{ + return read_const_counter(NULL); +} + +static struct clocksource clocksource_const = { + .name = "Constant", + .rating = 400, + .read = read_const_counter, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .vdso_clock_mode = VDSO_CLOCKMODE_CPU, +}; + +int __init constant_clocksource_init(void) +{ + int res; + unsigned long freq = const_clock_freq; + + res = clocksource_register_hz(&clocksource_const, freq); + + sched_clock_register(native_sched_clock, 64, freq); + + pr_info("Constant clock source device register\n"); + + return res; +} + +void __init time_init(void) +{ + if (!cpu_has_cpucfg) + const_clock_freq = cpu_clock_freq; + else + const_clock_freq = calc_const_freq(); + + init_offset = -(drdtime() - csr_read64(LOONGARCH_CSR_CNTC)); + + constant_clockevent_init(); + constant_clocksource_init(); + + pv_time_init(); +} diff --git a/arch/loongarch/kernel/topology.c b/arch/loongarch/kernel/topology.c new file mode 100644 index 0000000000000000000000000000000000000000..d5ea7a08487ed3ed9731078dd5c8b7219779579d --- /dev/null +++ b/arch/loongarch/kernel/topology.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include + +static DEFINE_PER_CPU(struct cpu, cpu_devices); + +#ifdef CONFIG_HOTPLUG_CPU +int arch_register_cpu(int cpu) +{ + int ret; + struct cpu *c = &per_cpu(cpu_devices, cpu); + + c->hotpluggable = 1; + ret = register_cpu(c, cpu); + if (ret < 0) + pr_warn("register_cpu %d failed (%d)\n", cpu, ret); + + return ret; +} +EXPORT_SYMBOL(arch_register_cpu); + +void arch_unregister_cpu(int cpu) +{ + struct cpu *c = &per_cpu(cpu_devices, cpu); + + c->hotpluggable = 0; + unregister_cpu(c); +} +EXPORT_SYMBOL(arch_unregister_cpu); +#endif + +static int __init topology_init(void) +{ + int i, ret; + +#ifdef CONFIG_NUMA + for_each_online_node(i) + register_one_node(i); +#endif /* CONFIG_NUMA */ + + for_each_present_cpu(i) { + struct cpu *c = &per_cpu(cpu_devices, i); + + c->hotpluggable = !!i; + ret = register_cpu(c, i); + if (ret < 0) + pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret); + } + + return 0; +} + +subsys_initcall(topology_init); diff --git a/arch/loongarch/kernel/trans_i386_syscalls.c b/arch/loongarch/kernel/trans_i386_syscalls.c new file mode 100644 index 0000000000000000000000000000000000000000..a71313d72367ee8d06cba0c723f3df7a9f4d1c90 --- /dev/null +++ b/arch/loongarch/kernel/trans_i386_syscalls.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Authors: Hanlu Li + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include + +typedef struct { + uint32_t sig[2]; +} latx_sigset_t; + +typedef uint32_t latx_size_t; + +static inline void latx_sig_setmask(sigset_t *blocked, old_sigset_t set) +{ + memcpy(blocked->sig, &set, sizeof(set)); +} + +/* This is for latx-i386 */ +SYSCALL_DEFINE4(latx_rt_sigprocmask, int, how, latx_sigset_t __user *, nset, + latx_sigset_t __user *, oset, latx_size_t, sigsetsize) +{ + old_sigset_t old_set, new_set; + sigset_t new_blocked; + + /* XXX: Don't preclude handling different sized sigset_t's. */ + if (sigsetsize != sizeof(latx_sigset_t)) + return -EINVAL; + + old_set = current->blocked.sig[0]; + + if (nset) { + if (copy_from_user(&new_set, nset, sizeof(latx_sigset_t))) + return -EFAULT; + new_set &= ~(sigmask(SIGKILL) | sigmask(SIGSTOP)); + + new_blocked = current->blocked; + + switch (how) { + case SIG_BLOCK: + sigaddsetmask(&new_blocked, new_set); + break; + case SIG_UNBLOCK: + sigdelsetmask(&new_blocked, new_set); + break; + case SIG_SETMASK: + latx_sig_setmask(&new_blocked, new_set); + break; + default: + return -EINVAL; + } + set_current_blocked(&new_blocked); + + if (copy_to_user(nset, &(current->blocked), + sizeof(latx_sigset_t))) + return -EFAULT; + } + + if (oset) { + if (copy_to_user(oset, &old_set, sizeof(latx_sigset_t))) + return -EFAULT; + } + + return 0; +} diff --git a/arch/loongarch/kernel/trans_mips_syscalls.c b/arch/loongarch/kernel/trans_mips_syscalls.c new file mode 100644 index 0000000000000000000000000000000000000000..ea8d319edbbfcef3a2f73605cede08e8162c15a9 --- /dev/null +++ b/arch/loongarch/kernel/trans_mips_syscalls.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Authors: Hanlu Li + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include +#include + +static const unsigned int mips_errno[] = { + [EDEADLK] = M_EDEADLK, + [ENAMETOOLONG] = M_ENAMETOOLONG, + [ENOLCK] = M_ENOLCK, + [ENOSYS] = M_ENOSYS, + [ENOTEMPTY] = M_ENOTEMPTY, + [ELOOP] = M_ELOOP, + [ENOMSG] = M_ENOMSG, + [EIDRM] = M_EIDRM, + [ECHRNG] = M_ECHRNG, + [EL2NSYNC] = M_EL2NSYNC, + [EL3HLT] = M_EL3HLT, + [EL3RST] = M_EL3RST, + [ELNRNG] = M_ELNRNG, + [EUNATCH] = M_EUNATCH, + [ENOCSI] = M_ENOCSI, + [EL2HLT] = M_EL2HLT, + [EBADE] = M_EBADE, + [EBADR] = M_EBADR, + [EXFULL] = M_EXFULL, + [ENOANO] = M_ENOANO, + [EBADRQC] = M_EBADRQC, + [EBADSLT] = M_EBADSLT, + [EBFONT] = M_EBFONT, + [ENOSTR] = M_ENOSTR, + [ENODATA] = M_ENODATA, + [ETIME] = M_ETIME, + [ENOSR] = M_ENOSR, + [ENONET] = M_ENONET, + [ENOPKG] = M_ENOPKG, + [EREMOTE] = M_EREMOTE, + [ENOLINK] = M_ENOLINK, + [EADV] = M_EADV, + [ESRMNT] = M_ESRMNT, + [ECOMM] = M_ECOMM, + [EPROTO] = M_EPROTO, + [EMULTIHOP] = M_EMULTIHOP, + [EDOTDOT] = M_EDOTDOT, + [EBADMSG] = M_EBADMSG, + [EOVERFLOW] = M_EOVERFLOW, + [ENOTUNIQ] = M_ENOTUNIQ, + [EBADFD] = M_EBADFD, + [EREMCHG] = M_EREMCHG, + [ELIBACC] = M_ELIBACC, + [ELIBBAD] = M_ELIBBAD, + [ELIBSCN] = M_ELIBSCN, + [ELIBMAX] = M_ELIBMAX, + [ELIBEXEC] = M_ELIBEXEC, + [EILSEQ] = M_EILSEQ, + [ERESTART] = M_ERESTART, + [ESTRPIPE] = M_ESTRPIPE, + [EUSERS] = M_EUSERS, + [ENOTSOCK] = M_ENOTSOCK, + [EDESTADDRREQ] = M_EDESTADDRREQ, + [EMSGSIZE] = M_EMSGSIZE, + [EPROTOTYPE] = M_EPROTOTYPE, + [ENOPROTOOPT] = M_ENOPROTOOPT, + [EPROTONOSUPPORT] = M_EPROTONOSUPPORT, + [ESOCKTNOSUPPORT] = M_ESOCKTNOSUPPORT, + [EOPNOTSUPP] = M_EOPNOTSUPP, + [EPFNOSUPPORT] = M_EPFNOSUPPORT, + [EAFNOSUPPORT] = M_EAFNOSUPPORT, + [EADDRINUSE] = M_EADDRINUSE, + [EADDRNOTAVAIL] = M_EADDRNOTAVAIL, + [ENETDOWN] = M_ENETDOWN, + [ENETUNREACH] = M_ENETUNREACH, + [ENETRESET] = M_ENETRESET, + [ECONNABORTED] = M_ECONNABORTED, + [ECONNRESET] = M_ECONNRESET, + [ENOBUFS] = M_ENOBUFS, + [EISCONN] = M_EISCONN, + [ENOTCONN] = M_ENOTCONN, + [ESHUTDOWN] = M_ESHUTDOWN, + [ETOOMANYREFS] = M_ETOOMANYREFS, + [ETIMEDOUT] = M_ETIMEDOUT, + [ECONNREFUSED] = M_ECONNREFUSED, + [EHOSTDOWN] = M_EHOSTDOWN, + [EHOSTUNREACH] = M_EHOSTUNREACH, + [EALREADY] = M_EALREADY, + [EINPROGRESS] = M_EINPROGRESS, + [ESTALE] = M_ESTALE, + [EUCLEAN] = M_EUCLEAN, + [ENOTNAM] = M_ENOTNAM, + [ENAVAIL] = M_ENAVAIL, + [EISNAM] = M_EISNAM, + [EREMOTEIO] = M_EREMOTEIO, + [EDQUOT] = M_EDQUOT, + [ENOMEDIUM] = M_ENOMEDIUM, + [EMEDIUMTYPE] = M_EMEDIUMTYPE, + [ECANCELED] = M_ECANCELED, + [ENOKEY] = M_ENOKEY, + [EKEYEXPIRED] = M_EKEYEXPIRED, + [EKEYREVOKED] = M_EKEYREVOKED, + [EKEYREJECTED] = M_EKEYREJECTED, + [EOWNERDEAD] = M_EOWNERDEAD, + [ENOTRECOVERABLE] = M_ENOTRECOVERABLE, + [ERFKILL] = M_ERFKILL, + [EHWPOISON] = M_EHWPOISON, +}; + +asmlinkage int trans_mips_errno(int errno) +{ + if (errno >= EDEADLK && errno <= EHWPOISON) + return mips_errno[errno]; + return errno; +} + +SYSCALL_DEFINE0(set_user_tp) +{ + current_thread_info()->tp_value = task_pt_regs(current)->regs[2]; + return 0; +} diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c new file mode 100644 index 0000000000000000000000000000000000000000..ce0e18d7335b5336270e7d1478809843a63be839 --- /dev/null +++ b/arch/loongarch/kernel/traps.c @@ -0,0 +1,1159 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "access-helper.h" + +void *exception_table[EXCCODE_INT_START] = { + [0 ... EXCCODE_INT_START - 1] = handle_reserved, + + [EXCCODE_TLBI] = handle_tlb_load, + [EXCCODE_TLBL] = handle_tlb_load, + [EXCCODE_TLBS] = handle_tlb_store, + [EXCCODE_TLBM] = handle_tlb_modify, + [EXCCODE_TLBNR] = handle_tlb_protect, + [EXCCODE_TLBNX] = handle_tlb_protect, + [EXCCODE_TLBPE] = handle_tlb_protect, + [EXCCODE_ADE] = handle_ade, + [EXCCODE_ALE] = handle_ale, + [EXCCODE_BCE] = handle_bce, + [EXCCODE_SYS] = handle_sys, + [EXCCODE_BP] = handle_bp, + [EXCCODE_INE] = handle_ri, + [EXCCODE_IPE] = handle_ri, + [EXCCODE_FPDIS] = handle_fpu, + [EXCCODE_LSXDIS] = handle_lsx, + [EXCCODE_LASXDIS] = handle_lasx, + [EXCCODE_FPE] = handle_fpe, + [EXCCODE_BTDIS] = handle_lbt, + [EXCCODE_WATCH] = handle_watch, +}; +EXPORT_SYMBOL_GPL(exception_table); + +static void show_backtrace(struct task_struct *task, const struct pt_regs *regs, + const char *loglvl, bool user) +{ + unsigned long pc; + struct unwind_state state; + struct pt_regs *pregs = (struct pt_regs *)regs; + + if (!task) + task = current; + + unwind_start(&state, task, pregs); + +#ifdef CONFIG_UNWINDER_PROLOGUE + if (user_mode(regs)) + state.enable = false; +#endif + + printk("%sCall Trace:\n", loglvl); + for (; !unwind_done(&state); unwind_next_frame(&state)) { + pc = unwind_get_return_address(&state); + print_ip_sym(loglvl, pc); + } + printk("%s\n", loglvl); +} + +static void show_stacktrace(struct task_struct *task, + const struct pt_regs *regs, const char *loglvl, bool user) +{ + int i; + const int field = 2 * sizeof(unsigned long); + unsigned long stackdata; + unsigned long *sp = (unsigned long *)regs->regs[3]; + + printk("%sStack :", loglvl); + i = 0; + while ((unsigned long) sp & (PAGE_SIZE - 1)) { + if (i && ((i % (64 / field)) == 0)) { + pr_cont("\n"); + printk("%s ", loglvl); + } + if (i > 39) { + pr_cont(" ..."); + break; + } + + if (__get_addr(&stackdata, sp++, user)) { + pr_cont(" (Bad stack address)"); + break; + } + + pr_cont(" %0*lx", field, stackdata); + i++; + } + pr_cont("\n"); + show_backtrace(task, regs, loglvl, user); +} + +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) +{ + struct pt_regs regs; + + regs.csr_crmd = 0; + if (sp) { + regs.csr_era = 0; + regs.regs[1] = 0; + regs.regs[3] = (unsigned long)sp; + } else { + if (!task || task == current) + prepare_frametrace(®s); + else { + regs.csr_era = task->thread.reg01; + regs.regs[1] = 0; + regs.regs[3] = task->thread.reg03; + regs.regs[22] = task->thread.reg22; + } + } + + show_stacktrace(task, ®s, loglvl, false); +} + +static void show_code(unsigned int *pc, bool user) +{ + long i; + unsigned int insn; + + printk("Code:"); + + for(i = -3 ; i < 6 ; i++) { + if (__get_inst(&insn, pc + i, user)) { + pr_cont(" (Bad address in era)\n"); + break; + } + pr_cont("%c%08x%c", (i?' ':'<'), insn, (i?' ':'>')); + } + pr_cont("\n"); +} + +static void print_bool_fragment(const char *key, unsigned long val, bool first) +{ + /* e.g. "+PG", "-DA" */ + pr_cont("%s%c%s", first ? "" : " ", val ? '+' : '-', key); +} + +static void print_plv_fragment(const char *key, int val) +{ + /* e.g. "PLV0", "PPLV3" */ + pr_cont("%s%d", key, val); +} + +static void print_memory_type_fragment(const char *key, unsigned long val) +{ + const char *humanized_type; + + switch (val) { + case 0: + humanized_type = "SUC"; + break; + case 1: + humanized_type = "CC"; + break; + case 2: + humanized_type = "WUC"; + break; + default: + pr_cont(" %s=Reserved(%lu)", key, val); + return; + } + + /* e.g. " DATM=WUC" */ + pr_cont(" %s=%s", key, humanized_type); +} + +static void print_intr_fragment(const char *key, unsigned long val) +{ + /* e.g. "LIE=0-1,3,5-7" */ + pr_cont("%s=%*pbl", key, EXCCODE_INT_NUM, &val); +} + +static void print_crmd(unsigned long x) +{ + printk(" CRMD: %08lx (", x); + print_plv_fragment("PLV", (int) FIELD_GET(CSR_CRMD_PLV, x)); + print_bool_fragment("IE", FIELD_GET(CSR_CRMD_IE, x), false); + print_bool_fragment("DA", FIELD_GET(CSR_CRMD_DA, x), false); + print_bool_fragment("PG", FIELD_GET(CSR_CRMD_PG, x), false); + print_memory_type_fragment("DACF", FIELD_GET(CSR_CRMD_DACF, x)); + print_memory_type_fragment("DACM", FIELD_GET(CSR_CRMD_DACM, x)); + print_bool_fragment("WE", FIELD_GET(CSR_CRMD_WE, x), false); + pr_cont(")\n"); +} + +static void print_prmd(unsigned long x) +{ + printk(" PRMD: %08lx (", x); + print_plv_fragment("PPLV", (int) FIELD_GET(CSR_PRMD_PPLV, x)); + print_bool_fragment("PIE", FIELD_GET(CSR_PRMD_PIE, x), false); + print_bool_fragment("PWE", FIELD_GET(CSR_PRMD_PWE, x), false); + pr_cont(")\n"); +} + +static void print_euen(unsigned long x) +{ + printk(" EUEN: %08lx (", x); + print_bool_fragment("FPE", FIELD_GET(CSR_EUEN_FPEN, x), true); + print_bool_fragment("SXE", FIELD_GET(CSR_EUEN_LSXEN, x), false); + print_bool_fragment("ASXE", FIELD_GET(CSR_EUEN_LASXEN, x), false); + print_bool_fragment("BTE", FIELD_GET(CSR_EUEN_LBTEN, x), false); + pr_cont(")\n"); +} + +static void print_ecfg(unsigned long x) +{ + printk(" ECFG: %08lx (", x); + print_intr_fragment("LIE", FIELD_GET(CSR_ECFG_IM, x)); + pr_cont(" VS=%d)\n", (int) FIELD_GET(CSR_ECFG_VS, x)); +} + +static const char *humanize_exc_name(unsigned int ecode, unsigned int esubcode) +{ + /* + * LoongArch users and developers are probably more familiar with + * those names found in the ISA manual, so we are going to print out + * the latter. This will require some mapping. + */ + switch (ecode) { + case EXCCODE_RSV: return "INT"; + case EXCCODE_TLBL: return "PIL"; + case EXCCODE_TLBS: return "PIS"; + case EXCCODE_TLBI: return "PIF"; + case EXCCODE_TLBM: return "PME"; + case EXCCODE_TLBNR: return "PNR"; + case EXCCODE_TLBNX: return "PNX"; + case EXCCODE_TLBPE: return "PPI"; + case EXCCODE_ADE: + switch (esubcode) { + case EXSUBCODE_ADEF: return "ADEF"; + case EXSUBCODE_ADEM: return "ADEM"; + } + break; + case EXCCODE_ALE: return "ALE"; + case EXCCODE_BCE: return "BCE"; + case EXCCODE_SYS: return "SYS"; + case EXCCODE_BP: return "BRK"; + case EXCCODE_INE: return "INE"; + case EXCCODE_IPE: return "IPE"; + case EXCCODE_FPDIS: return "FPD"; + case EXCCODE_LSXDIS: return "SXD"; + case EXCCODE_LASXDIS: return "ASXD"; + case EXCCODE_FPE: + switch (esubcode) { + case EXCSUBCODE_FPE: return "FPE"; + case EXCSUBCODE_VFPE: return "VFPE"; + } + break; + case EXCCODE_WATCH: + switch (esubcode) { + case EXCSUBCODE_WPEF: return "WPEF"; + case EXCSUBCODE_WPEM: return "WPEM"; + } + break; + case EXCCODE_BTDIS: return "BTD"; + case EXCCODE_BTE: return "BTE"; + case EXCCODE_GSPR: return "GSPR"; + case EXCCODE_HVC: return "HVC"; + case EXCCODE_GCM: + switch (esubcode) { + case EXCSUBCODE_GCSC: return "GCSC"; + case EXCSUBCODE_GCHC: return "GCHC"; + } + break; + /* + * The manual did not mention the EXCCODE_SE case, but print out it + * nevertheless. + */ + case EXCCODE_SE: return "SE"; + } + + return "???"; +} + +static void print_estat(unsigned long x) +{ + unsigned int ecode = FIELD_GET(CSR_ESTAT_EXC, x); + unsigned int esubcode = FIELD_GET(CSR_ESTAT_ESUBCODE, x); + + printk("ESTAT: %08lx [%s] (", x, humanize_exc_name(ecode, esubcode)); + print_intr_fragment("IS", FIELD_GET(CSR_ESTAT_IS, x)); + pr_cont(" ECode=%d EsubCode=%d)\n", (int) ecode, (int) esubcode); +} + +static void __show_regs(const struct pt_regs *regs) +{ + const int field = 2 * sizeof(unsigned long); + unsigned int exccode = FIELD_GET(CSR_ESTAT_EXC, regs->csr_estat); + + show_regs_print_info(KERN_DEFAULT); + + /* Print saved GPRs except $zero (substituting with PC/ERA) */ +#define GPR_FIELD(x) field, regs->regs[x] + printk("pc %0*lx ra %0*lx tp %0*lx sp %0*lx\n", + field, regs->csr_era, GPR_FIELD(1), GPR_FIELD(2), GPR_FIELD(3)); + printk("a0 %0*lx a1 %0*lx a2 %0*lx a3 %0*lx\n", + GPR_FIELD(4), GPR_FIELD(5), GPR_FIELD(6), GPR_FIELD(7)); + printk("a4 %0*lx a5 %0*lx a6 %0*lx a7 %0*lx\n", + GPR_FIELD(8), GPR_FIELD(9), GPR_FIELD(10), GPR_FIELD(11)); + printk("t0 %0*lx t1 %0*lx t2 %0*lx t3 %0*lx\n", + GPR_FIELD(12), GPR_FIELD(13), GPR_FIELD(14), GPR_FIELD(15)); + printk("t4 %0*lx t5 %0*lx t6 %0*lx t7 %0*lx\n", + GPR_FIELD(16), GPR_FIELD(17), GPR_FIELD(18), GPR_FIELD(19)); + printk("t8 %0*lx u0 %0*lx s9 %0*lx s0 %0*lx\n", + GPR_FIELD(20), GPR_FIELD(21), GPR_FIELD(22), GPR_FIELD(23)); + printk("s1 %0*lx s2 %0*lx s3 %0*lx s4 %0*lx\n", + GPR_FIELD(24), GPR_FIELD(25), GPR_FIELD(26), GPR_FIELD(27)); + printk("s5 %0*lx s6 %0*lx s7 %0*lx s8 %0*lx\n", + GPR_FIELD(28), GPR_FIELD(29), GPR_FIELD(30), GPR_FIELD(31)); + + /* The slot for $zero is reused as the syscall restart flag */ + if (regs->regs[0]) + printk("syscall restart flag: %0*lx\n", GPR_FIELD(0)); + + if (user_mode(regs)) { + printk(" ra: %0*lx\n", GPR_FIELD(1)); + printk(" ERA: %0*lx\n", field, regs->csr_era); + } else { + printk(" ra: %0*lx %pS\n", GPR_FIELD(1), (void *) regs->regs[1]); + printk(" ERA: %0*lx %pS\n", field, regs->csr_era, (void *) regs->csr_era); + } +#undef GPR_FIELD + + /* Print saved important CSRs */ + print_crmd(regs->csr_crmd); + print_prmd(regs->csr_prmd); + print_euen(regs->csr_euen); + print_ecfg(regs->csr_ecfg); + print_estat(regs->csr_estat); + + if (exccode >= EXCCODE_TLBL && exccode <= EXCCODE_ALE) + printk(" BADV: %0*lx\n", field, regs->csr_badvaddr); + + printk(" PRID: %08x (%s, %s)\n", read_cpucfg(LOONGARCH_CPUCFG0), + cpu_family_string(), cpu_full_name_string()); +} + +void show_regs(struct pt_regs *regs) +{ + __show_regs((struct pt_regs *)regs); + dump_stack(); +} + +void show_registers(struct pt_regs *regs) +{ + __show_regs(regs); + print_modules(); + printk("Process %s (pid: %d, threadinfo=%p, task=%p)\n", + current->comm, current->pid, current_thread_info(), current); + + show_stacktrace(current, regs, KERN_DEFAULT, user_mode(regs)); + show_code((void *)regs->csr_era, user_mode(regs)); + printk("\n"); +} + +static DEFINE_RAW_SPINLOCK(die_lock); + +void __noreturn die(const char *str, struct pt_regs *regs) +{ + static int die_counter; + int sig = SIGSEGV; + + oops_enter(); + + if (notify_die(DIE_OOPS, str, regs, 0, current->thread.trap_nr, + SIGSEGV) == NOTIFY_STOP) + sig = 0; + + console_verbose(); + raw_spin_lock_irq(&die_lock); + bust_spinlocks(1); + + printk("%s[#%d]:\n", str, ++die_counter); + show_registers(regs); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + raw_spin_unlock_irq(&die_lock); + + oops_exit(); + + if (regs && kexec_should_crash(current)) + crash_kexec(regs); + + if (in_interrupt()) + panic("Fatal exception in interrupt"); + + if (panic_on_oops) + panic("Fatal exception"); + + do_exit(sig); +} + +static inline void setup_vint_size(unsigned int size) +{ + unsigned int vs; + + vs = ilog2(size/4); + + if(vs == 0 || vs > 7) + panic("vint_size %d Not support yet", vs); + + csr_xchg32(vs<mm); + vma = find_vma(current->mm, (unsigned long)fault_addr); + if (vma && (vma->vm_start <= (unsigned long)fault_addr)) + si_code = SEGV_ACCERR; + else + si_code = SEGV_MAPERR; + mmap_read_unlock(current->mm); + force_sig_fault(SIGSEGV, si_code, fault_addr); + return 1; + + default: + force_sig(sig); + return 1; + } +} + +/* + * Delayed fp exceptions when doing a lazy ctx switch + */ +asmlinkage void noinstr do_fpe(struct pt_regs *regs, unsigned long fcsr) +{ + int sig; + void __user *fault_addr; + irqentry_state_t state = irqentry_enter(regs); + + if (notify_die(DIE_FP, "FP exception", regs, 0, current->thread.trap_nr, + SIGFPE) == NOTIFY_STOP) + goto out; + + /* Clear FCSR.Cause before enabling interrupts */ + write_fcsr(LOONGARCH_FCSR0, fcsr & ~mask_fcsr_x(fcsr)); + local_irq_enable(); + + die_if_kernel("FP exception in kernel code", regs); + + sig = SIGFPE; + fault_addr = (void __user *) regs->csr_era; + + /* Send a signal if required. */ + process_fpemu_return(sig, fault_addr, fcsr); + +out: + local_irq_disable(); + irqentry_exit(regs, state); +} + +asmlinkage void noinstr do_ade(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + die_if_kernel("Kernel ade access", regs); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)regs->csr_badvaddr); + + irqentry_exit(regs, state); +} + +/* sysctl hooks */ +int unaligned_enabled __read_mostly = 1; /* Enabled by default */ +int no_unaligned_warning __read_mostly = 1; /* Only 1 warning by default */ + +asmlinkage void noinstr do_ale(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + +#ifndef CONFIG_ARCH_STRICT_ALIGN + die_if_kernel("Kernel ale access", regs); + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); +#else + unsigned int *pc; + + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->csr_badvaddr); + + /* + * Did we catch a fault trying to load an instruction? + */ + if (regs->csr_badvaddr == regs->csr_era) + goto sigbus; + if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) + goto sigbus; + if (!unaligned_enabled) + goto sigbus; + if (!no_unaligned_warning) + show_registers(regs); + + pc = (unsigned int *)exception_era(regs); + + emulate_load_store_insn(regs, (void __user *)regs->csr_badvaddr, pc); + + goto out; + +sigbus: + die_if_kernel("Kernel ale access", regs); + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)regs->csr_badvaddr); +out: +#endif + irqentry_exit(regs, state); +} + +#ifdef CONFIG_GENERIC_BUG +int is_valid_bugaddr(unsigned long addr) +{ + return 1; +} +#endif /* CONFIG_GENERIC_BUG */ + +static void bug_handler(struct pt_regs *regs) +{ + switch (report_bug(regs->csr_era, regs)) { + case BUG_TRAP_TYPE_BUG: + case BUG_TRAP_TYPE_NONE: + die_if_kernel("Oops - BUG", regs); + force_sig(SIGTRAP); + break; + + case BUG_TRAP_TYPE_WARN: + /* Skip the BUG instruction and continue */ + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + } +} + +asmlinkage void noinstr do_bce(struct pt_regs *regs) +{ + bool user = user_mode(regs); + unsigned long era = exception_era(regs); + u64 badv = 0, lower = 0, upper = ULONG_MAX; + union loongarch_instruction insn; + irqentry_state_t state = irqentry_enter(regs); + + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_enable(); + + current->thread.trap_nr = read_csr_excode(); + + die_if_kernel("Bounds check error in kernel code", regs); + + /* + * Pull out the address that failed bounds checking, and the lower / + * upper bound, by minimally looking at the faulting instruction word + * and reading from the correct register. + */ + if (__get_inst(&insn.word, (u32 *)era, user)) + goto bad_era; + + switch (insn.reg3_format.opcode) { + case asrtled_op: + if (insn.reg3_format.rd != 0) + break; /* not asrtle */ + badv = regs->regs[insn.reg3_format.rj]; + upper = regs->regs[insn.reg3_format.rk]; + break; + + case asrtgtd_op: + if (insn.reg3_format.rd != 0) + break; /* not asrtgt */ + badv = regs->regs[insn.reg3_format.rj]; + lower = regs->regs[insn.reg3_format.rk]; + break; + + case ldleb_op: + case ldleh_op: + case ldlew_op: + case ldled_op: + case stleb_op: + case stleh_op: + case stlew_op: + case stled_op: + case fldles_op: + case fldled_op: + case fstles_op: + case fstled_op: + badv = regs->regs[insn.reg3_format.rj]; + upper = regs->regs[insn.reg3_format.rk]; + break; + + case ldgtb_op: + case ldgth_op: + case ldgtw_op: + case ldgtd_op: + case stgtb_op: + case stgth_op: + case stgtw_op: + case stgtd_op: + case fldgts_op: + case fldgtd_op: + case fstgts_op: + case fstgtd_op: + badv = regs->regs[insn.reg3_format.rj]; + lower = regs->regs[insn.reg3_format.rk]; + break; + } + + force_sig_bnderr((void __user *)badv, (void __user *)lower, (void __user *)upper); + +out: + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_disable(); + + irqentry_exit(regs, state); + return; + +bad_era: + /* + * Cannot pull out the instruction word, hence cannot provide more + * info than a regular SIGSEGV in this case. + */ + force_sig(SIGSEGV); + goto out; +} + +asmlinkage void noinstr do_bp(struct pt_regs *regs) +{ + bool user = user_mode(regs); + unsigned int opcode, bcode; + unsigned long era = exception_era(regs); + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + current->thread.trap_nr = read_csr_excode(); + if (__get_inst(&opcode, (u32 *)era, user)) + goto out_sigsegv; + + bcode = (opcode & 0x7fff); + +#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP + if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr, + SIGTRAP) == NOTIFY_STOP) + return; +#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ + + /* + * notify the kprobe handlers, if instruction is likely to + * pertain to them. + */ + switch (bcode) { + case BRK_KPROBE_BP: + if (notify_die(DIE_BREAK, "Kprobe", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + case BRK_KPROBE_SSTEPBP: + if (notify_die(DIE_SSTEPBP, "Kprobe_SingleStep", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + case BRK_UPROBE_BP: + if (notify_die(DIE_UPROBE, "Uprobe", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + case BRK_UPROBE_XOLBP: + if (notify_die(DIE_UPROBE_XOL, "Uprobe_XOL", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + default: + if (notify_die(DIE_TRAP, "Break", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + } + + switch (bcode) { + case BRK_BUG: + bug_handler(regs); + break; + case BRK_DIVZERO: + die_if_kernel("Break instruction in kernel code", regs); + force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->csr_era); + break; + case BRK_OVERFLOW: + die_if_kernel("Break instruction in kernel code", regs); + force_sig_fault(SIGFPE, FPE_INTOVF, (void __user *)regs->csr_era); + break; + default: + die_if_kernel("Break instruction in kernel code", regs); + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->csr_era); + break; + } + +out: + local_irq_disable(); + irqentry_exit(regs, state); + return; + +out_sigsegv: + force_sig(SIGSEGV); + goto out; +} + +asmlinkage void noinstr do_watch(struct pt_regs *regs) +{ + int i; + siginfo_t info; + irqentry_state_t state = irqentry_enter(regs); + struct loongarch_watch_reg_state *watches = ¤t->thread.watch; + + if (test_tsk_thread_flag(current, TIF_LOAD_WATCH)) { + loongarch_read_watch_registers(regs); + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + if ((watch_csrrd(LOONGARCH_CSR_MWPS) & (0x1 << i))) { + info.si_addr = (void __user *)watches->addr[i]; + watch_csrwr(0x1 << i, LOONGARCH_CSR_MWPS); + } + } + force_sig_fault(SIGTRAP, TRAP_HWBKPT, info.si_addr); + } else if (test_tsk_thread_flag(current, TIF_SINGLESTEP)) { + int llbit = (csr_read32(LOONGARCH_CSR_LLBCTL) & 0x1); + unsigned long pc = regs->csr_era; + + if (llbit) { + csr_write32(0x10000, LOONGARCH_CSR_FWPS); + csr_write32(0x4, LOONGARCH_CSR_LLBCTL); + } else if (pc == current->thread.single_step) { + csr_write32(0x10000, LOONGARCH_CSR_FWPS); + } else { + loongarch_read_watch_registers(regs); + force_sig(SIGTRAP); + } + } else { + if (notify_die(DIE_TRAP, "Break", regs, 0, + current->thread.trap_nr, SIGTRAP) != NOTIFY_STOP) + loongarch_clear_watch_registers(); + } + + irqentry_exit(regs, state); +} + +asmlinkage void noinstr do_ri(struct pt_regs *regs) +{ + int status = SIGILL; + unsigned int __maybe_unused opcode; + unsigned int __user *era = (unsigned int __user *)exception_era(regs); + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + current->thread.trap_nr = read_csr_excode(); + + if (notify_die(DIE_RI, "RI Fault", regs, 0, current->thread.trap_nr, + SIGILL) == NOTIFY_STOP) + goto out; + + die_if_kernel("Reserved instruction in kernel code", regs); + + if (unlikely(get_user(opcode, era) < 0)) { + status = SIGSEGV; + current->thread.error_code = 1; + } + + force_sig(status); + +out: + local_irq_disable(); + irqentry_exit(regs, state); +} + +static void init_restore_fp(void) +{ + if (!used_math()) { + /* First time FP context user. */ + init_fpu(); + } else { + /* This task has formerly used the FP context */ + if (!is_fpu_owner()) + own_fpu_inatomic(1); + } + + BUG_ON(!is_fp_enabled()); +} + +static void init_restore_lsx(void) +{ + enable_lsx(); + + if (!thread_lsx_context_live()) { + /* First time LSX context user */ + init_restore_fp(); + init_lsx_upper(); + set_thread_flag(TIF_LSX_CTX_LIVE); + } else { + if (!is_simd_owner()) { + if (is_fpu_owner()) { + restore_lsx_upper(current); + } else { + __own_fpu(); + restore_lsx(current); + } + } + } + + set_thread_flag(TIF_USEDSIMD); + + BUG_ON(!is_fp_enabled()); + BUG_ON(!is_lsx_enabled()); +} + +static void init_restore_lasx(void) +{ + enable_lasx(); + + if (!thread_lasx_context_live()) { + /* First time LASX context user */ + init_restore_lsx(); + init_lasx_upper(); + set_thread_flag(TIF_LASX_CTX_LIVE); + } else { + if (is_fpu_owner() || is_simd_owner()) { + init_restore_lsx(); + restore_lasx_upper(current); + } else { + __own_fpu(); + enable_lsx(); + restore_lasx(current); + } + } + + set_thread_flag(TIF_USEDSIMD); + + BUG_ON(!is_fp_enabled()); + BUG_ON(!is_lsx_enabled()); + BUG_ON(!is_lasx_enabled()); +} + +asmlinkage void noinstr do_fpu(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + die_if_kernel("do_fpu invoked from kernel context!", regs); + BUG_ON(is_lsx_enabled()); + BUG_ON(is_lasx_enabled()); + + preempt_disable(); + init_restore_fp(); + preempt_enable(); + + local_irq_disable(); + irqentry_exit(regs, state); +} + +asmlinkage void noinstr do_lsx(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + if (!cpu_has_lsx) { + force_sig(SIGILL); + goto out; + } + + die_if_kernel("do_lsx invoked from kernel context!", regs); + BUG_ON(is_lasx_enabled()); + + preempt_disable(); + init_restore_lsx(); + preempt_enable(); + +out: + local_irq_disable(); + irqentry_exit(regs, state); +} + +asmlinkage void noinstr do_lasx(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + if (!cpu_has_lasx) { + force_sig(SIGILL); + goto out; + } + + die_if_kernel("do_lasx invoked from kernel context!", regs); + + preempt_disable(); + init_restore_lasx(); + preempt_enable(); + +out: + local_irq_disable(); + irqentry_exit(regs, state); +} + +static void init_restore_lbt(void) +{ + if (!thread_lbt_context_live()) { + /* First lbt context user */ + init_lbt(); + set_thread_flag(TIF_LBT_CTX_LIVE); + } else { + /* Enable and restore */ + own_lbt_inatomic(1); + } +} + +asmlinkage void noinstr do_lbt(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_enable(); + + if (!cpu_has_lbt) { + force_sig(SIGILL); + goto out; + } + + preempt_disable(); + init_restore_lbt(); + preempt_enable(); + +out: + if (regs->csr_prmd & CSR_PRMD_PIE) + local_irq_disable(); + + irqentry_exit(regs, state); +} + +asmlinkage void noinstr do_reserved(struct pt_regs *regs) +{ + irqentry_state_t state = irqentry_enter(regs); + + local_irq_enable(); + /* + * Game over - no way to handle this if it ever occurs. Most probably + * caused by a fatal error after another hardware/software error. + */ + pr_err("Caught reserved exception %u on pid:%d [%s] - should not happen\n", + read_csr_excode(), current->pid, current->comm); + die_if_kernel("do_reserved exception", regs); + force_sig(SIGUNUSED); + + local_irq_disable(); + irqentry_exit(regs, state); +} + +asmlinkage void cache_parity_error(void) +{ + /* For the moment, report the problem and hang. */ + printk("Cache error exception:\n"); + printk("csr_merrctl == %08x\n", csr_read32(LOONGARCH_CSR_MERRCTL)); + printk("csr_merrera == %016lx\n", csr_read64(LOONGARCH_CSR_MERRERA)); + panic("Can't handle the cache error!"); +} + +asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp) +{ + register int cpu; + register unsigned long stack; + irqentry_state_t state = irqentry_enter(regs); + + cpu = smp_processor_id(); + + if (on_irq_stack(cpu, sp)) + handle_arch_irq(regs); + else { + stack = per_cpu(irq_stack, cpu) + IRQ_STACK_START; + + /* Save task's sp on IRQ stack for unwinding */ + *(unsigned long *)stack = sp; + + __asm__ __volatile__( + "move $s0, $sp \n" /* Preserve sp */ + "move $sp, %[stk] \n" /* Switch stack */ + "move $a0, %[regs] \n" + "la $t0, handle_arch_irq \n" + "ld.d $t1, $t0, 0 \n" + "jirl $ra, $t1, 0 \n" + "move $sp, $s0 \n" /* Restore sp */ + : /* No outputs */ + : [stk] "r" (stack), [regs] "r" (regs) + : "$a0", "$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7", "$s0", + "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8", + "memory"); + } + + irqentry_exit(regs, state); +} + +unsigned long eentry; +EXPORT_SYMBOL_GPL(eentry); +unsigned long tlbrentry; +EXPORT_SYMBOL_GPL(tlbrentry); + +long exception_handlers[VECSIZE * 128 / sizeof(long)] __aligned(SZ_64K); + +static void configure_exception_vector(void) +{ + eentry = (unsigned long)exception_handlers; + tlbrentry = (unsigned long)exception_handlers + 80*VECSIZE; + + csr_write64(eentry, LOONGARCH_CSR_EENTRY); + csr_write64(eentry, LOONGARCH_CSR_MERRENTRY); + csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY); +} + +void per_cpu_trap_init(int cpu) +{ + unsigned int i; + + setup_vint_size(VECSIZE); + + configure_exception_vector(); + + if (!cpu_data[cpu].asid_cache) + cpu_data[cpu].asid_cache = asid_first_version(cpu); + + mmgrab(&init_mm); + current->active_mm = &init_mm; + BUG_ON(current->mm); + enter_lazy_tlb(&init_mm, current); + + /* Initialise exception handlers */ + if (cpu == 0) + for (i = 0; i < 64; i++) + set_handler(i * VECSIZE, handle_reserved, VECSIZE); + + tlb_init(cpu); + cpu_cache_init(); +} + +/* Install CPU exception handler */ +void set_handler(unsigned long offset, void *addr, unsigned long size) +{ + memcpy((void *)(eentry + offset), addr, size); + local_flush_icache_range(eentry + offset, eentry + offset + size); +} + +static const char panic_null_cerr[] = + "Trying to set NULL cache error exception handler\n"; + +/* + * Install uncached CPU exception handler. + * This is suitable only for the cache error exception which is the only + * exception handler that is being run uncached. + */ +void set_merr_handler(unsigned long offset, void *addr, unsigned long size) +{ + unsigned long uncached_eentry = TO_UNCACHE(__pa(eentry)); + + if (!addr) + panic(panic_null_cerr); + + memcpy((void *)(uncached_eentry + offset), addr, size); +} + +void __init trap_init(void) +{ + long i; + + /* Set interrupt vector handler */ + for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++) + set_handler(i * VECSIZE, handle_vint, VECSIZE); + + for (i = EXCCODE_ADE; i <= EXCCODE_BTDIS; i++) + set_handler(i * VECSIZE, exception_table[i], VECSIZE); + + cache_error_setup(); + + local_flush_icache_range(eentry, eentry + 0x400); +} diff --git a/arch/loongarch/kernel/unaligned.c b/arch/loongarch/kernel/unaligned.c new file mode 100644 index 0000000000000000000000000000000000000000..3abf163dda056b681494a74916570866a6f1ec7b --- /dev/null +++ b/arch/loongarch/kernel/unaligned.c @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Handle unaligned accesses by emulation. + * + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle + * Copyright (C) 1999 Silicon Graphics, Inc. + * Copyright (C) 2014 Imagination Technologies Ltd. + */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "access-helper.h" + +#ifdef CONFIG_DEBUG_FS +static u32 unaligned_instructions_user; +static u32 unaligned_instructions_kernel; +#endif + +static inline unsigned long read_fpr(unsigned int idx) +{ +#define READ_FPR(idx, __value) \ + __asm__ __volatile__("movfr2gr.d %0, $f"#idx"\n\t" : "=r"(__value)); + + unsigned long __value; + + switch (idx) { + case 0: + READ_FPR(0, __value); + break; + case 1: + READ_FPR(1, __value); + break; + case 2: + READ_FPR(2, __value); + break; + case 3: + READ_FPR(3, __value); + break; + case 4: + READ_FPR(4, __value); + break; + case 5: + READ_FPR(5, __value); + break; + case 6: + READ_FPR(6, __value); + break; + case 7: + READ_FPR(7, __value); + break; + case 8: + READ_FPR(8, __value); + break; + case 9: + READ_FPR(9, __value); + break; + case 10: + READ_FPR(10, __value); + break; + case 11: + READ_FPR(11, __value); + break; + case 12: + READ_FPR(12, __value); + break; + case 13: + READ_FPR(13, __value); + break; + case 14: + READ_FPR(14, __value); + break; + case 15: + READ_FPR(15, __value); + break; + case 16: + READ_FPR(16, __value); + break; + case 17: + READ_FPR(17, __value); + break; + case 18: + READ_FPR(18, __value); + break; + case 19: + READ_FPR(19, __value); + break; + case 20: + READ_FPR(20, __value); + break; + case 21: + READ_FPR(21, __value); + break; + case 22: + READ_FPR(22, __value); + break; + case 23: + READ_FPR(23, __value); + break; + case 24: + READ_FPR(24, __value); + break; + case 25: + READ_FPR(25, __value); + break; + case 26: + READ_FPR(26, __value); + break; + case 27: + READ_FPR(27, __value); + break; + case 28: + READ_FPR(28, __value); + break; + case 29: + READ_FPR(29, __value); + break; + case 30: + READ_FPR(30, __value); + break; + case 31: + READ_FPR(31, __value); + break; + default: + panic("unexpected idx '%d'", idx); + } +#undef READ_FPR + return __value; +} + +static inline void write_fpr(unsigned int idx, unsigned long value) +{ +#define WRITE_FPR(idx, value) \ + __asm__ __volatile__("movgr2fr.d $f"#idx", %0\n\t" :: "r"(value)); + + switch (idx) { + case 0: + WRITE_FPR(0, value); + break; + case 1: + WRITE_FPR(1, value); + break; + case 2: + WRITE_FPR(2, value); + break; + case 3: + WRITE_FPR(3, value); + break; + case 4: + WRITE_FPR(4, value); + break; + case 5: + WRITE_FPR(5, value); + break; + case 6: + WRITE_FPR(6, value); + break; + case 7: + WRITE_FPR(7, value); + break; + case 8: + WRITE_FPR(8, value); + break; + case 9: + WRITE_FPR(9, value); + break; + case 10: + WRITE_FPR(10, value); + break; + case 11: + WRITE_FPR(11, value); + break; + case 12: + WRITE_FPR(12, value); + break; + case 13: + WRITE_FPR(13, value); + break; + case 14: + WRITE_FPR(14, value); + break; + case 15: + WRITE_FPR(15, value); + break; + case 16: + WRITE_FPR(16, value); + break; + case 17: + WRITE_FPR(17, value); + break; + case 18: + WRITE_FPR(18, value); + break; + case 19: + WRITE_FPR(19, value); + break; + case 20: + WRITE_FPR(20, value); + break; + case 21: + WRITE_FPR(21, value); + break; + case 22: + WRITE_FPR(22, value); + break; + case 23: + WRITE_FPR(23, value); + break; + case 24: + WRITE_FPR(24, value); + break; + case 25: + WRITE_FPR(25, value); + break; + case 26: + WRITE_FPR(26, value); + break; + case 27: + WRITE_FPR(27, value); + break; + case 28: + WRITE_FPR(28, value); + break; + case 29: + WRITE_FPR(29, value); + break; + case 30: + WRITE_FPR(30, value); + break; + case 31: + WRITE_FPR(31, value); + break; + default: + panic("unexpected idx '%d'", idx); + } +#undef WRITE_FPR +} + +void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc) +{ + bool fp = false; + bool sign, write; + bool user = user_mode(regs); + unsigned int res, size = 0; + unsigned long value = 0; + union loongarch_instruction insn; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + __get_inst(&insn.word, pc, user); + + switch (insn.reg2i12_format.opcode) { + case ldh_op: + size = 2; + sign = true; + write = false; + break; + case ldhu_op: + size = 2; + sign = false; + write = false; + break; + case sth_op: + size = 2; + sign = true; + write = true; + break; + case ldw_op: + size = 4; + sign = true; + write = false; + break; + case ldwu_op: + size = 4; + sign = false; + write = false; + break; + case stw_op: + size = 4; + sign = true; + write = true; + break; + case ldd_op: + size = 8; + sign = true; + write = false; + break; + case std_op: + size = 8; + sign = true; + write = true; + break; + case flds_op: + size = 4; + fp = true; + sign = true; + write = false; + break; + case fsts_op: + size = 4; + fp = true; + sign = true; + write = true; + break; + case fldd_op: + size = 8; + fp = true; + sign = true; + write = false; + break; + case fstd_op: + size = 8; + fp = true; + sign = true; + write = true; + break; + } + + switch (insn.reg2i14_format.opcode) { + case ldptrw_op: + size = 4; + sign = true; + write = false; + break; + case stptrw_op: + size = 4; + sign = true; + write = true; + break; + case ldptrd_op: + size = 8; + sign = true; + write = false; + break; + case stptrd_op: + size = 8; + sign = true; + write = true; + break; + } + + switch (insn.reg3_format.opcode) { + case ldxh_op: + size = 2; + sign = true; + write = false; + break; + case ldxhu_op: + size = 2; + sign = false; + write = false; + break; + case stxh_op: + size = 2; + sign = true; + write = true; + break; + case ldxw_op: + size = 4; + sign = true; + write = false; + break; + case ldxwu_op: + size = 4; + sign = false; + write = false; + break; + case stxw_op: + size = 4; + sign = true; + write = true; + break; + case ldxd_op: + size = 8; + sign = true; + write = false; + break; + case stxd_op: + size = 8; + sign = true; + write = true; + break; + case fldxs_op: + size = 4; + fp = true; + sign = true; + write = false; + break; + case fstxs_op: + size = 4; + fp = true; + sign = true; + write = true; + break; + case fldxd_op: + size = 8; + fp = true; + sign = true; + write = false; + break; + case fstxd_op: + size = 8; + fp = true; + sign = true; + write = true; + break; + } + + if (!size) + goto sigbus; + if (user && !access_ok(addr, size)) + goto sigbus; + + if (!write) { + res = unaligned_read(addr, &value, size, sign); + if (res) + goto fault; + + /* Rd is the same field in any formats */ + if (!fp) + regs->regs[insn.reg3_format.rd] = value; + else { + if (is_fpu_owner()) + write_fpr(insn.reg3_format.rd, value); + else + set_fpr64(¤t->thread.fpu.fpr[insn.reg3_format.rd], 0, value); + } + } else { + /* Rd is the same field in any formats */ + if (!fp) + value = regs->regs[insn.reg3_format.rd]; + else { + if (is_fpu_owner()) + value = read_fpr(insn.reg3_format.rd); + else + value = get_fpr64(¤t->thread.fpu.fpr[insn.reg3_format.rd], 0); + } + + res = unaligned_write(addr, value, size); + if (res) + goto fault; + } + +#ifdef CONFIG_DEBUG_FS + if (user) + unaligned_instructions_user++; + else + unaligned_instructions_kernel++; +#endif + + compute_return_era(regs); + + return; + +fault: + /* Did we have an exception handler installed? */ + if (fixup_exception(regs)) + return; + + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGSEGV); + + return; + +sigbus: + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGBUS); + + return; +} + +#ifdef CONFIG_DEBUG_FS +static int __init debugfs_unaligned(void) +{ + struct dentry *d; + + d = debugfs_create_dir("loongarch", NULL); + + debugfs_create_u32("unaligned_instructions_user", + S_IRUGO, d, &unaligned_instructions_user); + debugfs_create_u32("unaligned_instructions_kernel", + S_IRUGO, d, &unaligned_instructions_kernel); + + return 0; +} +arch_initcall(debugfs_unaligned); +#endif diff --git a/arch/loongarch/kernel/unwind_guess.c b/arch/loongarch/kernel/unwind_guess.c new file mode 100644 index 0000000000000000000000000000000000000000..7d113e57e7fa9d6584629deef8b7e50c4744999a --- /dev/null +++ b/arch/loongarch/kernel/unwind_guess.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include + +#include +#include + +unsigned long unwind_get_return_address(struct unwind_state *state) +{ + if (unwind_done(state)) + return 0; + + return state->pc; +} +EXPORT_SYMBOL_GPL(unwind_get_return_address); + +bool unwind_next_frame(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + unsigned long addr; + + if (unwind_done(state)) + return false; + + if (state->first) + state->first = false; + + do { + for (state->sp += sizeof(unsigned long); + state->sp < info->end; + state->sp += sizeof(unsigned long)) { + addr = *(unsigned long *)(state->sp); + state->pc = unwind_graph_addr(state, addr, state->sp + 8); + if (__kernel_text_address(state->pc)) + return true; + } + + state->sp = info->next_sp; + + } while (!get_stack_info(state->sp, state->task, info)); + + return false; +} +EXPORT_SYMBOL_GPL(unwind_next_frame); + +void unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs) +{ + memset(state, 0, sizeof(*state)); + + state->task = task; + + state->sp = regs->regs[3]; + state->pc = regs->csr_era; + state->first = true; + + state->pc = unwind_graph_addr(state, state->pc, state->sp); + get_stack_info(state->sp, state->task, &state->stack_info); + + if (!unwind_done(state) && !__kernel_text_address(state->pc)) + unwind_next_frame(state); +} +EXPORT_SYMBOL_GPL(unwind_start); diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c new file mode 100644 index 0000000000000000000000000000000000000000..5febcd8c2da4f770caadc129596669c3b529e15c --- /dev/null +++ b/arch/loongarch/kernel/unwind_orc.c @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Based on arch/x86/kernel/unwind_orc.c + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define orc_warn(fmt, ...) \ + printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) + +#define orc_warn_current(args...) \ +({ \ + if (state->task == current) \ + orc_warn(args); \ +}) + +extern int __start_orc_unwind_ip[]; +extern int __stop_orc_unwind_ip[]; +extern struct orc_entry __start_orc_unwind[]; +extern struct orc_entry __stop_orc_unwind[]; + +static bool orc_init __ro_after_init; +static unsigned int lookup_num_blocks __ro_after_init; + +static int *cur_orc_ip_table = __start_orc_unwind_ip; +static struct orc_entry *cur_orc_table = __start_orc_unwind; + +static inline unsigned long orc_ip(const int *ip) +{ + return (unsigned long)ip + *ip; +} + +static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, + unsigned int num_entries, unsigned long ip) +{ + int *first = ip_table; + int *last = ip_table + num_entries - 1; + int *mid = first, *found = first; + + if (!num_entries) + return NULL; + + /* + * Do a binary range search to find the rightmost duplicate of a given + * starting address. Some entries are section terminators which are + * "weak" entries for ensuring there are no gaps. They should be + * ignored when they conflict with a real entry. + */ + while (first <= last) { + mid = first + ((last - first) / 2); + + if (orc_ip(mid) <= ip) { + found = mid; + first = mid + 1; + } else + last = mid - 1; + } + + return u_table + (found - ip_table); +} + +#ifdef CONFIG_MODULES +static struct orc_entry *orc_module_find(unsigned long ip) +{ + struct module *mod; + + mod = __module_address(ip); + if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) + return NULL; + return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, + mod->arch.num_orcs, ip); +} +#else +static struct orc_entry *orc_module_find(unsigned long ip) +{ + return NULL; +} +#endif + +/* Fake frame pointer entry -- used as a fallback for generated code */ +static struct orc_entry orc_fp_entry = { + .type = UNWIND_HINT_TYPE_CALL, + .sp_reg = ORC_REG_FP, + .sp_offset = 16, + .fp_reg = ORC_REG_PREV_SP, + .fp_offset = -16, + .ra_reg = ORC_REG_PREV_SP, + .ra_offset = -8, + .end = 0, +}; + +static struct orc_entry *orc_find(unsigned long ip) +{ + struct orc_entry *orc = NULL; + + /* Although exception occurs, CSR_ERA is not 0. */ + if (ip == 0) + return NULL; + + /* For non-init vmlinux addresses, use the fast lookup table: */ + if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { + unsigned int idx, start, stop; + + idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; + + if (unlikely((idx >= lookup_num_blocks - 1))) { + orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n", + idx, lookup_num_blocks, (void *)ip); + return NULL; + } + + start = orc_lookup[idx]; + stop = orc_lookup[idx + 1] + 1; + + if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || + (__start_orc_unwind + stop > __stop_orc_unwind))) { + orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n", + idx, lookup_num_blocks, start, stop, (void *)ip); + return NULL; + } + + return __orc_find(__start_orc_unwind_ip + start, + __start_orc_unwind + start, stop - start, ip); + } + + /* vmlinux .init slow lookup: */ + if (init_kernel_text(ip)) + return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, + __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); + + /* Module lookup: */ + orc = orc_module_find(ip); + if (orc) + return orc; + + return NULL; +} + +#ifdef CONFIG_MODULES +static DEFINE_MUTEX(sort_mutex); + +static void orc_sort_swap(void *_a, void *_b, int size) +{ + int delta = _b - _a; + int *a = _a, *b = _b, tmp; + struct orc_entry *orc_a, *orc_b; + struct orc_entry orc_tmp; + + /* Swap the .orc_unwind_ip entries: */ + tmp = *a; + *a = *b + delta; + *b = tmp - delta; + + /* Swap the corresponding .orc_unwind entries: */ + orc_a = cur_orc_table + (a - cur_orc_ip_table); + orc_b = cur_orc_table + (b - cur_orc_ip_table); + orc_tmp = *orc_a; + *orc_a = *orc_b; + *orc_b = orc_tmp; +} + +static int orc_sort_cmp(const void *_a, const void *_b) +{ + const int *a = _a, *b = _b; + unsigned long a_val = orc_ip(a); + unsigned long b_val = orc_ip(b); + struct orc_entry *orc_a; + + if (a_val > b_val) + return 1; + if (a_val < b_val) + return -1; + + /* + * The "weak" section terminator entries need to always be on the left + * to ensure the lookup code skips them in favor of real entries. + * These terminator entries exist to handle any gaps created by + * whitelisted .o files which didn't get objtool generation. + */ + orc_a = cur_orc_table + (a - cur_orc_ip_table); + return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; +} + +void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, + void *_orc, size_t orc_size) +{ + int *orc_ip = _orc_ip; + struct orc_entry *orc = _orc; + unsigned int num_entries = orc_ip_size / sizeof(int); + + WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || + orc_size % sizeof(*orc) != 0 || + num_entries != orc_size / sizeof(*orc)); + + /* + * The 'cur_orc_*' globals allow the orc_sort_swap() callback to + * associate an .orc_unwind_ip table entry with its corresponding + * .orc_unwind entry so they can both be swapped. + */ + mutex_lock(&sort_mutex); + cur_orc_ip_table = orc_ip; + cur_orc_table = orc; + sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap); + mutex_unlock(&sort_mutex); + + mod->arch.orc_unwind_ip = orc_ip; + mod->arch.orc_unwind = orc; + mod->arch.num_orcs = num_entries; +} +#endif + +void __init unwind_init(void) +{ + int i; + size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; + size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; + size_t num_entries = orc_ip_size / sizeof(int); + struct orc_entry *orc; + + if (!num_entries || orc_ip_size % sizeof(int) != 0 || + orc_size % sizeof(struct orc_entry) != 0 || + num_entries != orc_size / sizeof(struct orc_entry)) { + orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n"); + return; + } + + /* + * Note, the orc_unwind and orc_unwind_ip tables were already + * sorted at build time via the 'sorttable' tool. + * It's ready for binary search straight away, no need to sort it. + */ + + /* Initialize the fast lookup table: */ + lookup_num_blocks = orc_lookup_end - orc_lookup; + for (i = 0; i < lookup_num_blocks - 1; i++) { + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, + num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); + if (!orc) { + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); + return; + } + + orc_lookup[i] = orc - __start_orc_unwind; + } + + /* Initialize the ending block: */ + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, LOOKUP_STOP_IP); + if (!orc) { + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); + return; + } + orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; + + orc_init = true; +} + +static inline bool task_on_another_cpu(struct task_struct *task) +{ +#ifdef CONFIG_SMP + return task != current && task->on_cpu; +#else + return false; +#endif +} + +void unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs) +{ + memset(state, 0, sizeof(*state)); + state->task = task; + + if (!orc_init) + goto err; + + if (task_on_another_cpu(task)) + goto err; + + state->pc = regs->csr_era; + state->ra = regs->regs[1]; + state->sp = regs->regs[3]; + state->fp = regs->regs[22]; + + if (get_stack_info(state->sp, state->task, &state->stack_info)) + goto err; + + return; + +err: + state->error = true; + state->stack_info.type = STACK_TYPE_UNKNOWN; +} + +#define INSN_LINK_OFFSET 4 + +extern void *exception_table[]; +extern asmlinkage void handle_vint(void); +extern asmlinkage void handle_reserved(void); + +static inline unsigned long bt_address(unsigned long ra) +{ + extern unsigned long eentry; + + if (__kernel_text_address(ra)) + return ra; + + if (__module_text_address(ra)) + return ra; + + if (ra >= eentry && ra <= eentry + EXCCODE_INT_END * VECSIZE) { + unsigned long type = (ra - eentry) / VECSIZE; + unsigned long offset = (ra - eentry) % VECSIZE; + unsigned long func; + switch (type) { + case 0 ... EXCCODE_INT_START-1: + func = (unsigned long)exception_table[type]; + break; + case EXCCODE_INT_START ... EXCCODE_INT_END: + func = (unsigned long)handle_vint; + break; + default: + func = (unsigned long)handle_reserved; + return 0; + } + + return func + offset; + } + + return ra; +} + +static inline bool on_stack(struct stack_info *info, unsigned long addr, + size_t len) +{ + unsigned long begin = info->begin; + unsigned long end = info->end; + + return (info->type != STACK_TYPE_UNKNOWN && + addr >= begin && addr < end && addr + len > begin && addr + len <= end); +} + +static bool stack_access_ok(struct unwind_state *state, unsigned long addr, + size_t len) +{ + struct stack_info *info = &state->stack_info; + + if (!on_stack(info, addr, len) && + (get_stack_info(addr, state->task, info))) + return false; + + return true; +} + +static bool is_entry_func(unsigned long addr) +{ + extern u32 kernel_entry; + extern u32 kernel_entry_end; + + return addr >= (unsigned long)&kernel_entry && + addr < (unsigned long)&kernel_entry_end; +} + +bool unwind_next_frame(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + struct orc_entry *orc; + struct pt_regs *regs; + unsigned long *p, pc; + + if (unwind_done(state)) + return false; + + /* Don't let modules unload while we're reading their ORC data. */ + preempt_disable(); + + if (is_entry_func(state->pc)) + goto end; + + orc = orc_find(state->pc); + if (!orc) { + orc = &orc_fp_entry; + state->error = true; + } + + switch (orc->sp_reg) { + case ORC_REG_SP: + state->sp = state->sp + orc->sp_offset; + break; + case ORC_REG_FP: + state->sp = state->fp; + break; + default: + orc_warn("unknown SP base reg %d at %pB\n", orc->sp_reg, (void *)state->pc); + goto err; + } + + switch (orc->fp_reg) { + case ORC_REG_PREV_SP: + p = (unsigned long *)(state->sp + orc->fp_offset); + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) + goto err; + + state->fp = *p; + break; + case ORC_REG_UNDEFINED: + /* Nothing. */ + break; + default: + orc_warn("unknown FP base reg %d at %pB\n", orc->fp_reg, (void *)state->pc); + goto err; + } + + switch (orc->type) { + case UNWIND_HINT_TYPE_CALL: + if (orc->ra_reg == ORC_REG_PREV_SP) { + p = (unsigned long *)(state->sp + orc->ra_offset); + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) + goto err; + + pc = unwind_graph_addr(state, *p, state->sp); + pc -= INSN_LINK_OFFSET; + } else if (orc->ra_reg == ORC_REG_UNDEFINED) { + if (!state->ra || state->ra == state->pc) + goto err; + + pc = unwind_graph_addr(state, state->ra, state->sp); + pc -= INSN_LINK_OFFSET; + state->ra = 0; + } else { + orc_warn("unknown ra base reg %d at %pB\n", orc->ra_reg, (void *)state->pc); + goto err; + } + break; + case UNWIND_HINT_TYPE_REGS: + if (state->stack_info.type == STACK_TYPE_IRQ && state->sp == info->end) + regs = (struct pt_regs *)info->next_sp; + else + regs = (struct pt_regs *)state->sp; + + if (!stack_access_ok(state, (unsigned long)regs, sizeof(*regs))) + goto err; + + if ((info->end == (unsigned long)regs + sizeof(*regs)) && + !regs->regs[3] && !regs->regs[1]) + goto end; + + if (user_mode(regs)) + goto end; + + pc = regs->csr_era; + if (!__kernel_text_address(pc)) + goto err; + + state->sp = regs->regs[3]; + state->ra = regs->regs[1]; + state->fp = regs->regs[22]; + get_stack_info(state->sp, state->task, info); + + break; + default: + orc_warn("unknown .orc_unwind entry type %d at %pB\n", orc->type, (void *)state->pc); + goto err; + } + + state->pc = bt_address(pc); + if (!state->pc) { + pr_err("cannot find unwind pc at %pK\n", (void *)pc); + goto err; + } + + preempt_enable(); + return true; + +err: + state->error = true; + +end: + preempt_enable(); + state->stack_info.type = STACK_TYPE_UNKNOWN; + return false; +} + +unsigned long unwind_get_return_address(struct unwind_state *state) +{ + if (unwind_done(state)) + return 0; + + return __kernel_text_address(state->pc) ? state->pc : 0; +} diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c new file mode 100644 index 0000000000000000000000000000000000000000..2aa8337ace3cda9f9830d781041088eec1fc0416 --- /dev/null +++ b/arch/loongarch/kernel/unwind_prologue.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include + +#include +#include +#include + +unsigned long unwind_get_return_address(struct unwind_state *state) +{ + if (unwind_done(state)) + return 0; + + return state->pc; +} +EXPORT_SYMBOL_GPL(unwind_get_return_address); + +static inline bool is_stack_open_ins(union loongarch_instruction *ip) +{ + /* addi.d $sp, $sp, -imm */ + return ip->reg2i12_format.opcode == addid_op && + ip->reg2i12_format.rj == 3 && + ip->reg2i12_format.rd == 3 && + ip->reg2i12_format.simmediate < 0; +} + +static inline bool is_ra_save_ins(union loongarch_instruction *ip) +{ + /* st.d $ra, $sp, offset */ + return ip->reg2i12_format.opcode == std_op && + ip->reg2i12_format.rj == 3 && + ip->reg2i12_format.rd == 1; +} + +static inline bool is_branch_ins(union loongarch_instruction *ip) +{ + return is_branch_insn(*ip); +} + +static inline void unwind_state_fixup(struct unwind_state *state) +{ +#ifdef CONFIG_DYNAMIC_FTRACE + static unsigned long ftrace_case = (unsigned long)ftrace_call + 4; + + if (state->pc == ftrace_case) + state->is_ftrace = true; +#endif +} + +static bool unwind_by_prologue(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + union loongarch_instruction *ip, *ip_end; + unsigned long frame_size = 0, frame_ra = -1; + unsigned long size, offset, pc; + struct pt_regs *regs; + + if (state->sp >= info->end || state->sp < info->begin) + return false; + + if (state->is_ftrace) { + /* + * As we meet ftrace_regs_entry, reset first flag like first doing + * tracing, Prologue analysis will stop soon because PC is at entry. + */ + regs = (struct pt_regs *)state->sp; + state->pc = regs->csr_era; + state->ra = regs->regs[1]; + state->sp = regs->regs[3]; + state->first = true; + state->is_ftrace = false; + return true; + } + + /* + * When first is not set, the PC is a return address in the previous frame. + * We need to adjust its value in case overflow to the next symbol. + */ + pc = state->pc - (state->first ? 0 : LOONGARCH_INSN_SIZE); + if (!kallsyms_lookup_size_offset(pc, &size, &offset)) + return false; + + ip = (union loongarch_instruction *)(pc - offset); + ip_end = (union loongarch_instruction *)pc; + + while (ip < ip_end) { + if (is_stack_open_ins(ip)) { + frame_size = -ip->reg2i12_format.simmediate; + ip++; + break; + } + ip++; + } + + if (!frame_size) { + if (state->first) + goto first; + + return false; + } + + while (ip < ip_end) { + if (is_ra_save_ins(ip)) { + frame_ra = ip->reg2i12_format.simmediate; + break; + } + if (is_branch_ins(ip)) + break; + ip++; + } + + if (frame_ra < 0) { + if (state->first) { + state->sp = state->sp + frame_size; + goto first; + } + return false; + } + + if (state->first) + state->first = false; + + state->pc = *(unsigned long *)(state->sp + frame_ra); + state->sp = state->sp + frame_size; + goto out; + +first: + state->first = false; + if (state->pc == state->ra) + return false; + + state->pc = state->ra; + +out: + unwind_state_fixup(state); + return !!__kernel_text_address(state->pc); +} + +static bool unwind_by_guess(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + unsigned long addr; + + for (state->sp += sizeof(unsigned long); + state->sp < info->end; + state->sp += sizeof(unsigned long)) { + addr = *(unsigned long *)(state->sp); + + state->pc = unwind_graph_addr(state, addr, state->sp + 8); + if (__kernel_text_address(state->pc)) + return true; + } + + return false; +} + +bool unwind_next_frame(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + struct pt_regs *regs; + unsigned long pc; + + if (unwind_done(state)) + return false; + + do { + if (state->enable) { + if (unwind_by_prologue(state)) { + state->pc = unwind_graph_addr(state, state->pc, state->sp); + return true; + } + if (info->type == STACK_TYPE_IRQ && + info->end == state->sp) { + regs = (struct pt_regs *)info->next_sp; + pc = regs->csr_era; + if (user_mode(regs) || !__kernel_text_address(pc)) + goto out; + + state->pc = pc; + state->sp = regs->regs[3]; + state->ra = regs->regs[1]; + state->first = true; + get_stack_info(state->sp, state->task, info); + + return true; + } + } else { + if (state->first) + state->first = false; + + if (unwind_by_guess(state)) + return true; + } + + state->sp = info->next_sp; + + } while (!get_stack_info(state->sp, state->task, info)); + +out: + state->stack_info.type = STACK_TYPE_UNKNOWN; + return false; +} +EXPORT_SYMBOL_GPL(unwind_next_frame); + +void unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs) +{ + memset(state, 0, sizeof(*state)); + + if (__kernel_text_address(regs->csr_era)) { + state->enable = true; + } + + state->task = task; + state->pc = regs->csr_era; + state->sp = regs->regs[3]; + state->ra = regs->regs[1]; + state->first = true; + state->pc = unwind_graph_addr(state, state->pc, state->sp); + get_stack_info(state->sp, state->task, &state->stack_info); + + if (!unwind_done(state) && !__kernel_text_address(state->pc)) + unwind_next_frame(state); +} +EXPORT_SYMBOL_GPL(unwind_start); diff --git a/arch/loongarch/kernel/uprobes.c b/arch/loongarch/kernel/uprobes.c new file mode 100644 index 0000000000000000000000000000000000000000..c176f17e19c63d7f625f5778b6cd930a3b6dcad2 --- /dev/null +++ b/arch/loongarch/kernel/uprobes.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/** + * arch_uprobe_analyze_insn - instruction analysis including validity and fixups. + * @mm: the probed address space. + * @arch_uprobe: the probepoint information. + * @addr: virtual address at which to install the probepoint + * Return 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *aup, + struct mm_struct *mm, unsigned long addr) +{ + union loongarch_instruction insn; + + if (addr & 0x03) + return -EINVAL; + + insn.word = aup->insn[0]; + + if (insn.reg1i21_format.opcode == bceqz_op) { + pr_notice("Uprobes for bceqz and bcnez instructions are not" + "supported\n"); + return -EINVAL; + } + + if (is_branch_insn(insn) || is_pc_insn(insn)) { + aup->ixol[0] = larch_insn_gen_nop(); + } else { + aup->ixol[0] = aup->insn[0]; + } + + aup->ixol[1] = UPROBE_BRK_UPROBE_XOL; + + return 0; +} + +/** + * is_trap_insn - check if the instruction is a trap variant + * @insn: instruction to be checked. + * Returns true if @insn is a trap variant. + * + * This definition overrides the weak definition in kernel/events/uprobes.c. + * and is needed for the case where an architecture has multiple trap + * instructions (like PowerPC or MIPS). We treat BREAK just like the more + * modern conditional trap instructions. + */ +bool is_trap_insn(uprobe_opcode_t *insn) +{ + union loongarch_instruction inst; + + inst.word = *insn; + + return (inst.reg0i15_format.opcode == break_op); +} + +#define UPROBE_TRAP_NR ULONG_MAX +static int ss_none; + +/* + * arch_uprobe_pre_xol - prepare to execute out of line. + * @auprobe: the probepoint information. + * @regs: reflects the saved user state of current task. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + union loongarch_instruction insn; + struct uprobe_task *utask = current->utask; + + insn.word = aup->insn[0]; + + /* + * Here do emulation if it were branch or pc-relative insn. + */ + if (is_branch_insn(insn)) { + if (!simu_branch(regs, insn)) + return -EFAULT; + aup->resume_era = regs->csr_era; + } else if (is_pc_insn(insn)) { + if (!simu_pc(regs, insn)) + return -EFAULT; + aup->resume_era = regs->csr_era; + } else { + ss_none = 1; + } + utask->autask.saved_trap_nr = current->thread.trap_nr; + current->thread.trap_nr = UPROBE_TRAP_NR; + regs->csr_era = current->utask->xol_vaddr; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_nr = utask->autask.saved_trap_nr; + if (ss_none) + regs->csr_era = current->utask->vaddr + 4; + else + regs->csr_era = aup->resume_era; + + return 0; +} + +/* + * If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. It is assumed that anything + * like do_page_fault/do_trap/etc sets thread.trap_nr != -1. + * + * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr, + * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to + * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol(). + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *tsk) +{ + if (tsk->thread.trap_nr != UPROBE_TRAP_NR) + return true; + + return false; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct die_args *args = data; + struct pt_regs *regs = args->regs; + + /* regs == NULL is a kernel bug */ + if (WARN_ON(!regs)) + return NOTIFY_DONE; + + /* We are only interested in userspace traps */ + if (!user_mode(regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_UPROBE: + if (uprobe_pre_sstep_notifier(regs)) + return NOTIFY_STOP; + break; + case DIE_UPROBE_XOL: + if (uprobe_post_sstep_notifier(regs)) + return NOTIFY_STOP; + default: + break; + } + + return 0; +} + +/* + * This function gets called when XOL instruction either gets trapped or + * the thread has a fatal signal. Reset the instruction pointer to its + * probed address for the potential restart or for post mortem analysis. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *aup, + struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + instruction_pointer_set(regs, utask->vaddr); +} + +unsigned long arch_uretprobe_hijack_return_addr( + unsigned long trampoline_vaddr, struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->regs[1]; + + /* Replace the return address with the trampoline address */ + regs->regs[1] = trampoline_vaddr; + + return ra; +} + +/** + * set_swbp - store breakpoint at a given address. + * @auprobe: arch specific probepoint information. + * @mm: the probed process address space. + * @vaddr: the virtual address to insert the opcode. + * + * For mm @mm, store the breakpoint instruction at @vaddr. + * Return 0 (success) or a negative errno. + * + * This version overrides the weak version in kernel/events/uprobes.c. + */ +int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long vaddr) +{ + return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN); +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + unsigned long kaddr, kstart; + + /* Initialize the slot */ + kaddr = (unsigned long)kmap_atomic(page); + kstart = kaddr + (vaddr & ~PAGE_MASK); + memcpy((void *)kstart, src, len); + flush_icache_range(kstart, kstart + len); + kunmap_atomic((void *)kaddr); +} + +/** + * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs + * @regs: Reflects the saved state of the task after it has hit a breakpoint + * instruction. + * Return the address of the breakpoint instruction. + * + * This overrides the weak version in kernel/events/uprobes.c. + */ +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +/* + * See if the instruction can be emulated. + * Returns true if instruction was emulated, false otherwise. + * + * For now we always emulate so this function just returns 0. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + return 0; +} diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c new file mode 100644 index 0000000000000000000000000000000000000000..0a7dafc2ad355c58f212a71cfcf68018fe70276a --- /dev/null +++ b/arch/loongarch/kernel/vdso.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Huacai Chen + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +extern char vdso_start[], vdso_end[]; + +/* Kernel-provided data used by the VDSO. */ +static union { + u8 page[PAGE_SIZE]; + struct vdso_data data[CS_BASES]; +} generic_vdso_data __page_aligned_data; + +static union { + u8 page[LOONGARCH_VDSO_DATA_SIZE]; + struct loongarch_vdso_data vdata; +} loongarch_vdso_data __page_aligned_data; + +static struct page *vdso_pages[] = { NULL }; +struct vdso_data *vdso_data = generic_vdso_data.data; +struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata; + +static struct page *find_timens_vvar_page(struct vm_area_struct *vma); + +static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) +{ + current->mm->context.vdso = (void *)(new_vma->vm_start); + + return 0; +} + +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + unsigned long pfn; + struct page *timens_page = find_timens_vvar_page(vma); + + switch (vmf->pgoff) { + case VVAR_GENERIC_PAGE_OFFSET: + if (!timens_page) + pfn = sym_to_pfn(vdso_data); + else + pfn = page_to_pfn(timens_page); + break; +#ifdef CONFIG_TIME_NS + case VVAR_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace specific + * VVAR is mapped with the VVAR_GENERIC_PAGE_OFFSET and the real + * VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!timens_page) + return VM_FAULT_SIGBUS; + else + pfn = sym_to_pfn(vdso_data); + break; +#endif /* CONFIG_TIME_NS */ + case VVAR_LOONGARCH_PAGES_START ... VVAR_LOONGARCH_PAGES_END: + pfn = sym_to_pfn(&loongarch_vdso_data) + vmf->pgoff - VVAR_LOONGARCH_PAGES_START; + break; + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +struct loongarch_vdso_info vdso_info = { + .vdso = vdso_start, + .size = PAGE_SIZE, + .code_mapping = { + .name = "[vdso]", + .pages = vdso_pages, + .mremap = vdso_mremap, + }, + .data_mapping = { + .name = "[vvar]", + .fault = vvar_fault, + }, + .offset_sigreturn = vdso_offset_sigreturn, +}; + +static int __init init_vdso(void) +{ + unsigned long i, cpu, pfn; + + BUG_ON(!PAGE_ALIGNED(vdso_info.vdso)); + BUG_ON(!PAGE_ALIGNED(vdso_info.size)); + + for_each_possible_cpu(cpu) + vdso_pdata[cpu].node = cpu_to_node(cpu); + + pfn = __phys_to_pfn(__pa_symbol(vdso_info.vdso)); + for (i = 0; i < vdso_info.size / PAGE_SIZE; i++) + vdso_info.code_mapping.pages[i] = pfn_to_page(pfn + i); + + return 0; +} +subsys_initcall(init_vdso); + +#ifdef CONFIG_TIME_NS +struct vdso_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_data *)(vvar_page); +} + +/* + * The vvar mapping contains data for a specific time namespace, so when a + * task changes namespace we must unmap its vvar data for the old namespace. + * Subsequent faults will map in data for the new namespace. + * + * For more details see timens_setup_vdso_data(). + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + + mmap_read_lock(mm); + for (vma = mm->mmap; vma; vma = vma->vm_next) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, &vdso_info.data_mapping)) + zap_page_range(vma, vma->vm_start, size); + } + mmap_read_unlock(mm); + + return 0; +} + +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + if (likely(vma->vm_mm == current->mm)) + return current->nsproxy->time_ns->vvar_page; + + /* + * VM_PFNMAP | VM_IO protect .fault() handler from being called + * through interfaces like /proc/$pid/mem or + * process_vm_{readv,writev}() as long as there's no .access() + * in special_mapping_vmops. + * For more details check_vma_flags() and __access_remote_vm() + */ + WARN(1, "vvar_page accessed remotely"); + + return NULL; +} +#else +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + return NULL; +} +#endif + +static unsigned long vdso_base(void) +{ + unsigned long base = STACK_TOP; + + if (current->flags & PF_RANDOMIZE) { + base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1); + base = PAGE_ALIGN(base); + } + + return base; +} + +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + int ret; + unsigned long size, data_addr, vdso_addr; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + struct loongarch_vdso_info *info = current->thread.vdso; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + /* + * Determine total area size. This includes the VDSO data itself + * and the data pages. + */ + size = VVAR_SIZE + info->size; + + data_addr = get_unmapped_area(NULL, vdso_base(), size, 0, 0); + if (IS_ERR_VALUE(data_addr)) { + ret = data_addr; + goto out; + } + + vma = _install_special_mapping(mm, data_addr, VVAR_SIZE, + VM_READ | VM_MAYREAD | VM_PFNMAP, + &info->data_mapping); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out; + } + + vdso_addr = data_addr + VVAR_SIZE; + vma = _install_special_mapping(mm, vdso_addr, info->size, + VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, + &info->code_mapping); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out; + } + + mm->context.vdso = (void *)vdso_addr; + ret = 0; + +out: + mmap_write_unlock(mm); + return ret; +} diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S new file mode 100644 index 0000000000000000000000000000000000000000..cd973bcda25f93ae466813ef68757a8a0c62d987 --- /dev/null +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include + +#define PAGE_SIZE _PAGE_SIZE +#define RO_EXCEPTION_TABLE_ALIGN 4 +#define RUNTIME_DISCARD_EXIT + +/* + * Put .bss..swapper_pg_dir as the first thing in .bss. This will + * ensure that it has .bss alignment (64K). + */ +#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) + +#include + +OUTPUT_ARCH(loongarch) +ENTRY(kernel_entry) +PHDRS { + text PT_LOAD FLAGS(7); /* RWX */ + note PT_NOTE FLAGS(4); /* R__ */ +} + +jiffies = jiffies_64; + +SECTIONS +{ + . = VMLINUX_LOAD_ADDRESS; + /* Read-only */ + _text = .; /* Text and read-only data */ + .text : { + TEXT_TEXT + SCHED_TEXT + CPUIDLE_TEXT + LOCK_TEXT + KPROBES_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT + *(.fixup) + *(.gnu.warning) + } :text = 0 + _etext = .; /* End of text section */ + + /* + * struct alt_inst entries. From the header (alternative.h): + * "Alternative instructions for different CPU types or capabilities" + * Think locking instructions on spinlocks. + */ + . = ALIGN(4); + .altinstructions : AT(ADDR(.altinstructions) - LOAD_OFFSET) { + __alt_instructions = .; + *(.altinstructions) + __alt_instructions_end = .; + } + + .got : ALIGN(16) { *(.got) } + .plt : ALIGN(16) { *(.plt) } + .got.plt : ALIGN(16) { *(.got.plt) } + + _sdata = .; /* Start of data section */ + RO_DATA(4096) + RW_DATA(1 << CONFIG_L1_CACHE_SHIFT, PAGE_SIZE, THREAD_SIZE) + + ORC_UNWIND_TABLE + + /* We want the small data sections together, so single-instruction offsets + can access them all, and initialized data all before uninitialized, so + we can shorten the on-disk segment size. */ + .sdata : { + *(.sdata) + } + _edata = .; /* End of data section */ + + /* Will be freed after init */ + . = ALIGN(PAGE_SIZE); /* Init code and data */ + __init_begin = .; + INIT_TEXT_SECTION(PAGE_SIZE) + INIT_DATA_SECTION(16) + + . = ALIGN(4); + + /* .exit.text is discarded at runtime, not link time, to deal with + * references from .rodata + */ + .exit.text : { + EXIT_TEXT + } + .exit.data : { + EXIT_DATA + } +#ifdef CONFIG_SMP + PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT) +#endif + + .rela.dyn : ALIGN(8) { *(.rela.dyn) *(.rela*) } + +#ifdef CONFIG_RELOCATABLE + . = ALIGN(4); + + .data.reloc : { + _relocation_start = .; + /* + * Space for relocation table + * This needs to be filled so that the + * relocs tool can overwrite the content. + * An invalid value is left at the start of the + * section to abort relocation if the table + * has not been filled in. + */ + LONG(0xFFFFFFFF); + FILL(0); + . += CONFIG_RELOCATION_TABLE_SIZE - 4; + _relocation_end = .; + } +#endif + + /* + * Align to 64K in attempt to eliminate holes before the + * .bss..swapper_pg_dir section at the start of .bss. This + * also satisfies PAGE_SIZE alignment as the largest page size + * allowed is 64K. + */ + . = ALIGN(0x10000); + __init_end = .; + /* freed after init ends here */ + + /* + * Force .bss to 64K alignment so that .bss..swapper_pg_dir + * gets that alignment. .sbss should be empty, so there will be + * no holes after __init_end. */ + BSS_SECTION(0, 0x10000, 8) + + _end = . ; + + STABS_DEBUG + DWARF_DEBUG + ELF_DETAILS + + /* These must appear regardless of . */ + .gptab.sdata : { + *(.gptab.data) + *(.gptab.sdata) + } + .gptab.sbss : { + *(.gptab.bss) + *(.gptab.sbss) + } + + /* Sections to be discarded */ + DISCARDS + /DISCARD/ : { + *(.gnu.attributes) + *(.options) + *(.eh_frame) + } +} diff --git a/arch/loongarch/kernel/watch.c b/arch/loongarch/kernel/watch.c new file mode 100644 index 0000000000000000000000000000000000000000..4ef837388aaa9b5b52e777fb36ad83c29c4e9092 --- /dev/null +++ b/arch/loongarch/kernel/watch.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Chong Qiao + * Huacai Chen + * + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include + +#include +#include +#include + +unsigned long watch_csrrd(unsigned int reg) +{ + switch (reg) { + case LOONGARCH_CSR_IB0ADDR: + return csr_read64(LOONGARCH_CSR_IB0ADDR); + case LOONGARCH_CSR_IB1ADDR: + return csr_read64(LOONGARCH_CSR_IB1ADDR); + case LOONGARCH_CSR_IB2ADDR: + return csr_read64(LOONGARCH_CSR_IB2ADDR); + case LOONGARCH_CSR_IB3ADDR: + return csr_read64(LOONGARCH_CSR_IB3ADDR); + case LOONGARCH_CSR_IB4ADDR: + return csr_read64(LOONGARCH_CSR_IB4ADDR); + case LOONGARCH_CSR_IB5ADDR: + return csr_read64(LOONGARCH_CSR_IB5ADDR); + case LOONGARCH_CSR_IB6ADDR: + return csr_read64(LOONGARCH_CSR_IB6ADDR); + case LOONGARCH_CSR_IB7ADDR: + return csr_read64(LOONGARCH_CSR_IB7ADDR); + + case LOONGARCH_CSR_IB0MASK: + return csr_read64(LOONGARCH_CSR_IB0MASK); + case LOONGARCH_CSR_IB1MASK: + return csr_read64(LOONGARCH_CSR_IB1MASK); + case LOONGARCH_CSR_IB2MASK: + return csr_read64(LOONGARCH_CSR_IB2MASK); + case LOONGARCH_CSR_IB3MASK: + return csr_read64(LOONGARCH_CSR_IB3MASK); + case LOONGARCH_CSR_IB4MASK: + return csr_read64(LOONGARCH_CSR_IB4MASK); + case LOONGARCH_CSR_IB5MASK: + return csr_read64(LOONGARCH_CSR_IB5MASK); + case LOONGARCH_CSR_IB6MASK: + return csr_read64(LOONGARCH_CSR_IB6MASK); + case LOONGARCH_CSR_IB7MASK: + return csr_read64(LOONGARCH_CSR_IB7MASK); + + case LOONGARCH_CSR_IB0ASID: + return csr_read64(LOONGARCH_CSR_IB0ASID); + case LOONGARCH_CSR_IB1ASID: + return csr_read64(LOONGARCH_CSR_IB1ASID); + case LOONGARCH_CSR_IB2ASID: + return csr_read64(LOONGARCH_CSR_IB2ASID); + case LOONGARCH_CSR_IB3ASID: + return csr_read64(LOONGARCH_CSR_IB3ASID); + case LOONGARCH_CSR_IB4ASID: + return csr_read64(LOONGARCH_CSR_IB4ASID); + case LOONGARCH_CSR_IB5ASID: + return csr_read64(LOONGARCH_CSR_IB5ASID); + case LOONGARCH_CSR_IB6ASID: + return csr_read64(LOONGARCH_CSR_IB6ASID); + case LOONGARCH_CSR_IB7ASID: + return csr_read64(LOONGARCH_CSR_IB7ASID); + + case LOONGARCH_CSR_IB0CTL: + return csr_read64(LOONGARCH_CSR_IB0CTL); + case LOONGARCH_CSR_IB1CTL: + return csr_read64(LOONGARCH_CSR_IB1CTL); + case LOONGARCH_CSR_IB2CTL: + return csr_read64(LOONGARCH_CSR_IB2CTL); + case LOONGARCH_CSR_IB3CTL: + return csr_read64(LOONGARCH_CSR_IB3CTL); + case LOONGARCH_CSR_IB4CTL: + return csr_read64(LOONGARCH_CSR_IB4CTL); + case LOONGARCH_CSR_IB5CTL: + return csr_read64(LOONGARCH_CSR_IB5CTL); + case LOONGARCH_CSR_IB6CTL: + return csr_read64(LOONGARCH_CSR_IB6CTL); + case LOONGARCH_CSR_IB7CTL: + return csr_read64(LOONGARCH_CSR_IB7CTL); + + case LOONGARCH_CSR_DB0ADDR: + return csr_read64(LOONGARCH_CSR_DB0ADDR); + case LOONGARCH_CSR_DB1ADDR: + return csr_read64(LOONGARCH_CSR_DB1ADDR); + case LOONGARCH_CSR_DB2ADDR: + return csr_read64(LOONGARCH_CSR_DB2ADDR); + case LOONGARCH_CSR_DB3ADDR: + return csr_read64(LOONGARCH_CSR_DB3ADDR); + case LOONGARCH_CSR_DB4ADDR: + return csr_read64(LOONGARCH_CSR_DB4ADDR); + case LOONGARCH_CSR_DB5ADDR: + return csr_read64(LOONGARCH_CSR_DB5ADDR); + case LOONGARCH_CSR_DB6ADDR: + return csr_read64(LOONGARCH_CSR_DB6ADDR); + case LOONGARCH_CSR_DB7ADDR: + return csr_read64(LOONGARCH_CSR_DB7ADDR); + + case LOONGARCH_CSR_DB0MASK: + return csr_read64(LOONGARCH_CSR_DB0MASK); + case LOONGARCH_CSR_DB1MASK: + return csr_read64(LOONGARCH_CSR_DB1MASK); + case LOONGARCH_CSR_DB2MASK: + return csr_read64(LOONGARCH_CSR_DB2MASK); + case LOONGARCH_CSR_DB3MASK: + return csr_read64(LOONGARCH_CSR_DB3MASK); + case LOONGARCH_CSR_DB4MASK: + return csr_read64(LOONGARCH_CSR_DB4MASK); + case LOONGARCH_CSR_DB5MASK: + return csr_read64(LOONGARCH_CSR_DB5MASK); + case LOONGARCH_CSR_DB6MASK: + return csr_read64(LOONGARCH_CSR_DB6MASK); + case LOONGARCH_CSR_DB7MASK: + return csr_read64(LOONGARCH_CSR_DB7MASK); + + case LOONGARCH_CSR_DB0ASID: + return csr_read64(LOONGARCH_CSR_DB0ASID); + case LOONGARCH_CSR_DB1ASID: + return csr_read64(LOONGARCH_CSR_DB1ASID); + case LOONGARCH_CSR_DB2ASID: + return csr_read64(LOONGARCH_CSR_DB2ASID); + case LOONGARCH_CSR_DB3ASID: + return csr_read64(LOONGARCH_CSR_DB3ASID); + case LOONGARCH_CSR_DB4ASID: + return csr_read64(LOONGARCH_CSR_DB4ASID); + case LOONGARCH_CSR_DB5ASID: + return csr_read64(LOONGARCH_CSR_DB5ASID); + case LOONGARCH_CSR_DB6ASID: + return csr_read64(LOONGARCH_CSR_DB6ASID); + case LOONGARCH_CSR_DB7ASID: + return csr_read64(LOONGARCH_CSR_DB7ASID); + + case LOONGARCH_CSR_DB0CTL: + return csr_read64(LOONGARCH_CSR_DB0CTL); + case LOONGARCH_CSR_DB1CTL: + return csr_read64(LOONGARCH_CSR_DB1CTL); + case LOONGARCH_CSR_DB2CTL: + return csr_read64(LOONGARCH_CSR_DB2CTL); + case LOONGARCH_CSR_DB3CTL: + return csr_read64(LOONGARCH_CSR_DB3CTL); + case LOONGARCH_CSR_DB4CTL: + return csr_read64(LOONGARCH_CSR_DB4CTL); + case LOONGARCH_CSR_DB5CTL: + return csr_read64(LOONGARCH_CSR_DB5CTL); + case LOONGARCH_CSR_DB6CTL: + return csr_read64(LOONGARCH_CSR_DB6CTL); + case LOONGARCH_CSR_DB7CTL: + return csr_read64(LOONGARCH_CSR_DB7CTL); + + case LOONGARCH_CSR_FWPS: + return csr_read64(LOONGARCH_CSR_FWPS); + case LOONGARCH_CSR_MWPS: + return csr_read64(LOONGARCH_CSR_MWPS); + case LOONGARCH_CSR_FWPC: + return csr_read64(LOONGARCH_CSR_FWPC); + case LOONGARCH_CSR_MWPC: + return csr_read64(LOONGARCH_CSR_MWPC); + default: + printk("read watch register number error %d\n", reg); + } + return 0; +} +EXPORT_SYMBOL(watch_csrrd); + +void watch_csrwr(unsigned long val, unsigned int reg) +{ + switch (reg) { + case LOONGARCH_CSR_IB0ADDR: + csr_write64(val, LOONGARCH_CSR_IB0ADDR); + break; + case LOONGARCH_CSR_IB1ADDR: + csr_write64(val, LOONGARCH_CSR_IB1ADDR); + break; + case LOONGARCH_CSR_IB2ADDR: + csr_write64(val, LOONGARCH_CSR_IB2ADDR); + break; + case LOONGARCH_CSR_IB3ADDR: + csr_write64(val, LOONGARCH_CSR_IB3ADDR); + break; + case LOONGARCH_CSR_IB4ADDR: + csr_write64(val, LOONGARCH_CSR_IB4ADDR); + break; + case LOONGARCH_CSR_IB5ADDR: + csr_write64(val, LOONGARCH_CSR_IB5ADDR); + break; + case LOONGARCH_CSR_IB6ADDR: + csr_write64(val, LOONGARCH_CSR_IB6ADDR); + break; + case LOONGARCH_CSR_IB7ADDR: + csr_write64(val, LOONGARCH_CSR_IB7ADDR); + break; + + case LOONGARCH_CSR_IB0MASK: + csr_write64(val, LOONGARCH_CSR_IB0MASK); + break; + case LOONGARCH_CSR_IB1MASK: + csr_write64(val, LOONGARCH_CSR_IB1MASK); + break; + case LOONGARCH_CSR_IB2MASK: + csr_write64(val, LOONGARCH_CSR_IB2MASK); + break; + case LOONGARCH_CSR_IB3MASK: + csr_write64(val, LOONGARCH_CSR_IB3MASK); + break; + case LOONGARCH_CSR_IB4MASK: + csr_write64(val, LOONGARCH_CSR_IB4MASK); + break; + case LOONGARCH_CSR_IB5MASK: + csr_write64(val, LOONGARCH_CSR_IB5MASK); + break; + case LOONGARCH_CSR_IB6MASK: + csr_write64(val, LOONGARCH_CSR_IB6MASK); + break; + case LOONGARCH_CSR_IB7MASK: + csr_write64(val, LOONGARCH_CSR_IB7MASK); + break; + + case LOONGARCH_CSR_IB0ASID: + csr_write64(val, LOONGARCH_CSR_IB0ASID); + break; + case LOONGARCH_CSR_IB1ASID: + csr_write64(val, LOONGARCH_CSR_IB1ASID); + break; + case LOONGARCH_CSR_IB2ASID: + csr_write64(val, LOONGARCH_CSR_IB2ASID); + break; + case LOONGARCH_CSR_IB3ASID: + csr_write64(val, LOONGARCH_CSR_IB3ASID); + break; + case LOONGARCH_CSR_IB4ASID: + csr_write64(val, LOONGARCH_CSR_IB4ASID); + break; + case LOONGARCH_CSR_IB5ASID: + csr_write64(val, LOONGARCH_CSR_IB5ASID); + break; + case LOONGARCH_CSR_IB6ASID: + csr_write64(val, LOONGARCH_CSR_IB6ASID); + break; + case LOONGARCH_CSR_IB7ASID: + csr_write64(val, LOONGARCH_CSR_IB7ASID); + break; + + case LOONGARCH_CSR_IB0CTL: + csr_write64(val, LOONGARCH_CSR_IB0CTL); + break; + case LOONGARCH_CSR_IB1CTL: + csr_write64(val, LOONGARCH_CSR_IB1CTL); + break; + case LOONGARCH_CSR_IB2CTL: + csr_write64(val, LOONGARCH_CSR_IB2CTL); + break; + case LOONGARCH_CSR_IB3CTL: + csr_write64(val, LOONGARCH_CSR_IB3CTL); + break; + case LOONGARCH_CSR_IB4CTL: + csr_write64(val, LOONGARCH_CSR_IB4CTL); + break; + case LOONGARCH_CSR_IB5CTL: + csr_write64(val, LOONGARCH_CSR_IB5CTL); + break; + case LOONGARCH_CSR_IB6CTL: + csr_write64(val, LOONGARCH_CSR_IB6CTL); + break; + case LOONGARCH_CSR_IB7CTL: + csr_write64(val, LOONGARCH_CSR_IB7CTL); + break; + + case LOONGARCH_CSR_DB0ADDR: + csr_write64(val, LOONGARCH_CSR_DB0ADDR); + break; + case LOONGARCH_CSR_DB1ADDR: + csr_write64(val, LOONGARCH_CSR_DB1ADDR); + break; + case LOONGARCH_CSR_DB2ADDR: + csr_write64(val, LOONGARCH_CSR_DB2ADDR); + break; + case LOONGARCH_CSR_DB3ADDR: + csr_write64(val, LOONGARCH_CSR_DB3ADDR); + break; + case LOONGARCH_CSR_DB4ADDR: + csr_write64(val, LOONGARCH_CSR_DB4ADDR); + break; + case LOONGARCH_CSR_DB5ADDR: + csr_write64(val, LOONGARCH_CSR_DB5ADDR); + break; + case LOONGARCH_CSR_DB6ADDR: + csr_write64(val, LOONGARCH_CSR_DB6ADDR); + break; + case LOONGARCH_CSR_DB7ADDR: + csr_write64(val, LOONGARCH_CSR_DB7ADDR); + break; + + case LOONGARCH_CSR_DB0MASK: + csr_write64(val, LOONGARCH_CSR_DB0MASK); + break; + case LOONGARCH_CSR_DB1MASK: + csr_write64(val, LOONGARCH_CSR_DB1MASK); + break; + case LOONGARCH_CSR_DB2MASK: + csr_write64(val, LOONGARCH_CSR_DB2MASK); + break; + case LOONGARCH_CSR_DB3MASK: + csr_write64(val, LOONGARCH_CSR_DB3MASK); + break; + case LOONGARCH_CSR_DB4MASK: + csr_write64(val, LOONGARCH_CSR_DB4MASK); + break; + case LOONGARCH_CSR_DB5MASK: + csr_write64(val, LOONGARCH_CSR_DB5MASK); + break; + case LOONGARCH_CSR_DB6MASK: + csr_write64(val, LOONGARCH_CSR_DB6MASK); + break; + case LOONGARCH_CSR_DB7MASK: + csr_write64(val, LOONGARCH_CSR_DB7MASK); + break; + + case LOONGARCH_CSR_DB0ASID: + csr_write64(val, LOONGARCH_CSR_DB0ASID); + break; + case LOONGARCH_CSR_DB1ASID: + csr_write64(val, LOONGARCH_CSR_DB1ASID); + break; + case LOONGARCH_CSR_DB2ASID: + csr_write64(val, LOONGARCH_CSR_DB2ASID); + break; + case LOONGARCH_CSR_DB3ASID: + csr_write64(val, LOONGARCH_CSR_DB3ASID); + break; + case LOONGARCH_CSR_DB4ASID: + csr_write64(val, LOONGARCH_CSR_DB4ASID); + break; + case LOONGARCH_CSR_DB5ASID: + csr_write64(val, LOONGARCH_CSR_DB5ASID); + break; + case LOONGARCH_CSR_DB6ASID: + csr_write64(val, LOONGARCH_CSR_DB6ASID); + break; + case LOONGARCH_CSR_DB7ASID: + csr_write64(val, LOONGARCH_CSR_DB7ASID); + break; + + case LOONGARCH_CSR_DB0CTL: + csr_write64(val, LOONGARCH_CSR_DB0CTL); + break; + case LOONGARCH_CSR_DB1CTL: + csr_write64(val, LOONGARCH_CSR_DB1CTL); + break; + case LOONGARCH_CSR_DB2CTL: + csr_write64(val, LOONGARCH_CSR_DB2CTL); + break; + case LOONGARCH_CSR_DB3CTL: + csr_write64(val, LOONGARCH_CSR_DB3CTL); + break; + case LOONGARCH_CSR_DB4CTL: + csr_write64(val, LOONGARCH_CSR_DB4CTL); + break; + case LOONGARCH_CSR_DB5CTL: + csr_write64(val, LOONGARCH_CSR_DB5CTL); + break; + case LOONGARCH_CSR_DB6CTL: + csr_write64(val, LOONGARCH_CSR_DB6CTL); + break; + case LOONGARCH_CSR_DB7CTL: + csr_write64(val, LOONGARCH_CSR_DB7CTL); + break; + + case LOONGARCH_CSR_FWPS: + csr_write64(val, LOONGARCH_CSR_FWPS); + break; + case LOONGARCH_CSR_MWPS: + csr_write64(val, LOONGARCH_CSR_MWPS); + break; + default: + printk("write watch register number error %d\n", reg); + } +} +EXPORT_SYMBOL(watch_csrwr); + +void loongarch_probe_watch_registers(struct cpuinfo_loongarch *c) +{ + unsigned int ibcn, dbcn, total; + + ibcn = watch_csrrd(LOONGARCH_CSR_FWPC) & 0x3f; + dbcn = watch_csrrd(LOONGARCH_CSR_MWPC) & 0x3f; + c->watch_ireg_count = ibcn; + c->watch_dreg_count = dbcn; + total = ibcn + dbcn; + c->watch_reg_use_cnt = (total < NUM_WATCH_REGS) ? total : NUM_WATCH_REGS; + if (total) + c->options |= LOONGARCH_CPU_WATCH; +} + +/* + * Clear all watch registers. + */ +void loongarch_clear_watch_registers(void) +{ + unsigned int i, j, ibst, dbst, dbcn; + + ibst = watch_csrrd(LOONGARCH_CSR_FWPS); + dbst = watch_csrrd(LOONGARCH_CSR_MWPS); + dbcn = boot_cpu_data.watch_dreg_count; + + /* + * bit 16: skip next event + * bit 0-15: breakpoint triggered, W1C + */ + + if (ibst & 0x1ffff) + watch_csrwr(ibst&~0x10000, LOONGARCH_CSR_FWPS); + if (dbst & 0x1ffff) + watch_csrwr(dbst&~0x10000, LOONGARCH_CSR_MWPS); + + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + j = i - dbcn; + + if (i < dbcn) { + watch_csrwr(0, LOONGARCH_CSR_DB0ADDR + 8*i); + watch_csrwr(0, LOONGARCH_CSR_DB0MASK + 8*i); + watch_csrwr(0, LOONGARCH_CSR_DB0ASID + 8*i); + watch_csrwr(0, LOONGARCH_CSR_DB0CTL + 8*i); + } else { + watch_csrwr(0, LOONGARCH_CSR_IB0ADDR + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0MASK + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0ASID + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0CTL + 8*j); + } + } +} +EXPORT_SYMBOL(loongarch_clear_watch_registers); + +/* + * Install the watch registers for the current thread. + */ +void loongarch_install_watch_registers(struct task_struct *t) +{ + int i, j, dbcn; + unsigned int cntl = 0x1e; + struct loongarch_watch_reg_state *watches = &t->thread.watch; + + dbcn = boot_cpu_data.watch_dreg_count; + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + j = i - dbcn; + + if (watches->irw[i] & (LA_WATCH_R | LA_WATCH_W)) { + if ((watches->irw[i] & LA_WATCH_R)) + cntl |= 1<<8; + if ((watches->irw[i] & LA_WATCH_W)) + cntl |= 1<<9; + + watch_csrwr(watches->addr[i], LOONGARCH_CSR_DB0ADDR + 8*i); + watch_csrwr(watches->mask[i], LOONGARCH_CSR_DB0MASK + 8*i); + watch_csrwr(0, LOONGARCH_CSR_DB0ASID + 8*i); + watch_csrwr(cntl, LOONGARCH_CSR_DB0CTL + 8*i); + + } else if (watches->irw[i] & LA_WATCH_I) { + if ((task_pt_regs(t)->csr_era & ~watches->mask[i]) + == (watches->addr[i] & ~watches->mask[i])) + watch_csrwr(0x10000, LOONGARCH_CSR_FWPS); + + watch_csrwr(watches->addr[i], LOONGARCH_CSR_IB0ADDR + 8*j); + watch_csrwr(watches->mask[i], LOONGARCH_CSR_IB0MASK + 8*j); + watch_csrwr(0, LOONGARCH_CSR_IB0ASID + 8*j); + watch_csrwr(cntl, LOONGARCH_CSR_IB0CTL + 8*j); + } else if (watches->irwmask[i]) { + if (i < dbcn) + watch_csrwr(0x0, LOONGARCH_CSR_DB0CTL + 8*i); + else + watch_csrwr(0x0, LOONGARCH_CSR_IB0CTL + 8*j); + } + } +} +EXPORT_SYMBOL(loongarch_install_watch_registers); + +#define PRMD_WE (1UL<<3) /* PRMD watch enable, restore to CRMD when eret */ + +void loongarch_clear_prev_watch_registers(struct task_struct *prev) +{ + struct pt_regs *regs = task_pt_regs(prev); + + loongarch_clear_watch_registers(); + prev->thread.csr_prmd &= ~PRMD_WE; + regs->csr_prmd &= ~PRMD_WE; +} + +void loongarch_install_next_watch_registers(struct task_struct *next) +{ + struct pt_regs *regs = task_pt_regs(next); + + loongarch_install_watch_registers(next); + next->thread.csr_prmd |= PRMD_WE; + regs->csr_prmd |= PRMD_WE; +} + +/* + * Read back the watchhi registers so the user space debugger has + * access to the I, R, and W bits. + */ +void loongarch_read_watch_registers(struct pt_regs *regs) +{ + unsigned int i, j, ibst, dbst, dbc, dbcn; + struct loongarch_watch_reg_state *watches = ¤t->thread.watch; + + ibst = watch_csrrd(LOONGARCH_CSR_FWPS); + dbst = watch_csrrd(LOONGARCH_CSR_MWPS); + dbcn = boot_cpu_data.watch_dreg_count; + + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + j = i - dbcn; + watches->irwstat[i] = 0; + + if (i < dbcn) { + watches->addr[i] = watch_csrrd(LOONGARCH_CSR_DB0ADDR + 8*i); + watches->mask[i] = watch_csrrd(LOONGARCH_CSR_DB0MASK + 8*i); + if (dbst & (1u << i)) { + dbc = watch_csrrd(LOONGARCH_CSR_DB0CTL + i*8); + + if (dbc & (1<<8)) + watches->irwstat[i] |= LA_WATCH_R; + if (dbc & (1<<9)) + watches->irwstat[i] |= LA_WATCH_W; + } + } else { + watches->addr[i] = watch_csrrd(LOONGARCH_CSR_IB0ADDR + 8*j); + watches->mask[i] = watch_csrrd(LOONGARCH_CSR_IB0MASK + 8*j); + if (ibst & (1u << j)) + watches->irwstat[i] |= LA_WATCH_I; + } + } + + if (ibst & 0xffff) + watch_csrwr(ibst, LOONGARCH_CSR_FWPS); +} + +#define CTRL_WE 1 /* Breakpoint can only be written by hw debugger */ + +void loongarch_update_watch_registers(struct task_struct *t) +{ + int i, j, ctl, dbcn; + struct loongarch_watch_reg_state *watches = &t->thread.watch; + + dbcn = boot_cpu_data.watch_dreg_count; + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + j = i - dbcn; + + if (i < dbcn) { + ctl = watch_csrrd(LOONGARCH_CSR_DB0CTL + 8*i); + if (ctl & CTRL_WE) + watches->irwmask[i] = 0; + else + watches->irwmask[i] = LA_WATCH_R | LA_WATCH_W; + } else { + ctl = watch_csrrd(LOONGARCH_CSR_IB0CTL + 8*j); + if (ctl & CTRL_WE) + watches->irwmask[i] = 0; + else + watches->irwmask[i] = LA_WATCH_I; + } + } +} diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..c32e708b8961e6e963f6beab7fdc48c102e0aee1 --- /dev/null +++ b/arch/loongarch/kvm/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# KVM configuration +# +source "virt/kvm/Kconfig" + +menuconfig VIRTUALIZATION + bool "Virtualization" + help + Say Y here to get to see options for using your Linux host to run + other operating systems inside virtual machines (guests). + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if VIRTUALIZATION + +config KVM + tristate "Kernel-based Virtual Machine (KVM) support" + depends on HAVE_KVM + select PREEMPT_NOTIFIERS + select ANON_INODES + select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select HAVE_KVM_VCPU_ASYNC_IOCTL + select KVM_MMIO + select MMU_NOTIFIER + select HAVE_KVM_IRQCHIP + select HAVE_KVM_IRQFD + select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_EVENTFD + select HAVE_KVM_MSI + select SRCU + select KVM_VFIO + help + Support for hosting Guest kernels. This use the LoongArch + Virtualization (LVZ) ASE which supports running unmodified + guest kernels. + +source "drivers/vhost/Kconfig" + +endif # VIRTUALIZATION diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..4cd064d548ce3707b3bcd7f1e1b4b0106b90179a --- /dev/null +++ b/arch/loongarch/kvm/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for KVM support for LoongArch +# + +OBJECT_FILES_NON_STANDARD_entry.o := y + +ccflags-y := -Ivirt/kvm -Iarch/loongarch/kvm + +common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o \ + irqchip.o eventfd.o) + +KVM := ../../../virt/kvm +common-objs-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o + +kvm-objs := $(common-objs-y) loongarch.o emulate.o interrupt.o +kvm-objs += hypcall.o +kvm-objs += mmu.o + +kvm-objs += exit.o intc/ls7a_irq.o intc/ls3a_ipi.o intc/irqchip-debug.o\ + timer.o intc/ls3a_ext_irq.o irqfd.o csr.o +obj-$(CONFIG_KVM) += kvm.o +obj-y += entry.o fpu.o diff --git a/arch/loongarch/kvm/csr.c b/arch/loongarch/kvm/csr.c new file mode 100644 index 0000000000000000000000000000000000000000..6d2c89eebb1599d86d62f460a54930b4beac0455 --- /dev/null +++ b/arch/loongarch/kvm/csr.c @@ -0,0 +1,678 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include "kvmcpu.h" +#include "intc/ls3a_ipi.h" +#include "intc/ls3a_ext_irq.h" +#include "ls_irq.h" +#include "kvm_compat.h" +#include "kvmcsr.h" + +#define CASE_READ_SW_GCSR(csr, regid, csrid) \ + do { \ + if (regid == csrid) { \ + return kvm_read_sw_gcsr(csr, csrid); \ + } \ + } while (0) + +unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + unsigned long val = 0; + + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRCTL); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRERA); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRSAVE); + /* read sw csr when not config pmu to guest */ + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL0); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL3); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3); + + val = 0; + if (csrid < 4096) + val = kvm_read_sw_gcsr(csr, csrid); + else + pr_warn_once("Unsupport csrread 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); + + return val; +} + +#define CASE_WRITE_SW_GCSR(csr, regid, csrid, val) \ + do { \ + if (regid == csrid) { \ + kvm_write_sw_gcsr(csr, csrid, val); \ + return ; \ + } \ + } while (0) + +void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long val) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRCTL, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO1, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO2, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRSAVE, val); + + /* give pmu register to guest when config perfctrl */ + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL0, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL1, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL2, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL3, val); + /* write sw pmu csr if not config ctrl */ + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3, val); + + + if (csrid < 4096) + kvm_write_sw_gcsr(csr, csrid, val); + else + pr_warn_once("Unsupport csrwrite 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); +} + +#define CASE_CHANGE_SW_GCSR(csr, regid, csrid, mask, val) \ + do { \ + if (regid == csrid) { \ + kvm_change_sw_gcsr(csr, csrid, mask, val); \ + return ; \ + } \ + } while (0) + +void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long csr_mask, unsigned long val) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_IMPCTL1, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRCTL, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO1, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRINFO2, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRSAVE, csr_mask, val); + + if (csrid < 4096) { + unsigned long orig; + + orig = kvm_read_sw_gcsr(csr, csrid); + orig &= ~csr_mask; + orig |= val & csr_mask; + kvm_write_sw_gcsr(csr, csrid, orig); + } + pr_warn_once("Unsupport csrxchg 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); +} + +int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + GET_HW_GCSR(id, KVM_CSR_CRMD, v); + GET_HW_GCSR(id, KVM_CSR_PRMD, v); + GET_HW_GCSR(id, KVM_CSR_EUEN, v); + GET_HW_GCSR(id, KVM_CSR_MISC, v); + GET_HW_GCSR(id, KVM_CSR_ECFG, v); + GET_HW_GCSR(id, KVM_CSR_ESTAT, v); + GET_HW_GCSR(id, KVM_CSR_ERA, v); + GET_HW_GCSR(id, KVM_CSR_BADV, v); + GET_HW_GCSR(id, KVM_CSR_BADI, v); + GET_HW_GCSR(id, KVM_CSR_EENTRY, v); + GET_HW_GCSR(id, KVM_CSR_TLBIDX, v); + GET_HW_GCSR(id, KVM_CSR_TLBEHI, v); + GET_HW_GCSR(id, KVM_CSR_TLBELO0, v); + GET_HW_GCSR(id, KVM_CSR_TLBELO1, v); + GET_HW_GCSR(id, KVM_CSR_ASID, v); + GET_HW_GCSR(id, KVM_CSR_PGDL, v); + GET_HW_GCSR(id, KVM_CSR_PGDH, v); + GET_HW_GCSR(id, KVM_CSR_PWCTL0, v); + GET_HW_GCSR(id, KVM_CSR_PWCTL1, v); + GET_HW_GCSR(id, KVM_CSR_STLBPGSIZE, v); + GET_HW_GCSR(id, KVM_CSR_RVACFG, v); + GET_HW_GCSR(id, KVM_CSR_CPUID, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG1, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG2, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG3, v); + GET_HW_GCSR(id, KVM_CSR_KS0, v); + GET_HW_GCSR(id, KVM_CSR_KS1, v); + GET_HW_GCSR(id, KVM_CSR_KS2, v); + GET_HW_GCSR(id, KVM_CSR_KS3, v); + GET_HW_GCSR(id, KVM_CSR_KS4, v); + GET_HW_GCSR(id, KVM_CSR_KS5, v); + GET_HW_GCSR(id, KVM_CSR_KS6, v); + GET_HW_GCSR(id, KVM_CSR_KS7, v); + GET_HW_GCSR(id, KVM_CSR_TMID, v); + GET_HW_GCSR(id, KVM_CSR_TCFG, v); + GET_HW_GCSR(id, KVM_CSR_TVAL, v); + GET_HW_GCSR(id, KVM_CSR_CNTC, v); + GET_HW_GCSR(id, KVM_CSR_LLBCTL, v); + GET_HW_GCSR(id, KVM_CSR_TLBRENTRY, v); + GET_HW_GCSR(id, KVM_CSR_TLBRBADV, v); + GET_HW_GCSR(id, KVM_CSR_TLBRERA, v); + GET_HW_GCSR(id, KVM_CSR_TLBRSAVE, v); + GET_HW_GCSR(id, KVM_CSR_TLBRELO0, v); + GET_HW_GCSR(id, KVM_CSR_TLBRELO1, v); + GET_HW_GCSR(id, KVM_CSR_TLBREHI, v); + GET_HW_GCSR(id, KVM_CSR_TLBRPRMD, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN0, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN1, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN2, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN3, v); + GET_HW_GCSR(id, KVM_CSR_MWPS, v); + GET_HW_GCSR(id, KVM_CSR_FWPS, v); + + GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v); + GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRCTL, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRINFO1, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRINFO2, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRSAVE, v); + GET_SW_GCSR(csr, id, KVM_CSR_CTAG, v); + GET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v); + GET_SW_GCSR(csr, id, KVM_CSR_DERA, v); + GET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v); + + GET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v); + + if (force && (id < CSR_ALL_SIZE)) { + *v = kvm_read_sw_gcsr(csr, id); + return 0; + } + + return -1; +} + +int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + int ret; + + SET_HW_GCSR(csr, id, KVM_CSR_CRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_PRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_EUEN, v); + SET_HW_GCSR(csr, id, KVM_CSR_MISC, v); + SET_HW_GCSR(csr, id, KVM_CSR_ECFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_ERA, v); + SET_HW_GCSR(csr, id, KVM_CSR_BADV, v); + SET_HW_GCSR(csr, id, KVM_CSR_BADI, v); + SET_HW_GCSR(csr, id, KVM_CSR_EENTRY, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBIDX, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBEHI, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBELO0, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBELO1, v); + SET_HW_GCSR(csr, id, KVM_CSR_ASID, v); + SET_HW_GCSR(csr, id, KVM_CSR_PGDL, v); + SET_HW_GCSR(csr, id, KVM_CSR_PGDH, v); + SET_HW_GCSR(csr, id, KVM_CSR_PWCTL0, v); + SET_HW_GCSR(csr, id, KVM_CSR_PWCTL1, v); + SET_HW_GCSR(csr, id, KVM_CSR_STLBPGSIZE, v); + SET_HW_GCSR(csr, id, KVM_CSR_RVACFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_CPUID, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS0, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS1, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS2, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS3, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS4, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS5, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS6, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS7, v); + SET_HW_GCSR(csr, id, KVM_CSR_TMID, v); + SET_HW_GCSR(csr, id, KVM_CSR_TCFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_TVAL, v); + SET_HW_GCSR(csr, id, KVM_CSR_CNTC, v); + SET_HW_GCSR(csr, id, KVM_CSR_LLBCTL, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRENTRY, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRBADV, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRERA, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRSAVE, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO0, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO1, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBREHI, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRPRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN0, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN1, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN2, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN3, v); + SET_HW_GCSR(csr, id, KVM_CSR_MWPS, v); + SET_HW_GCSR(csr, id, KVM_CSR_FWPS, v); + + SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v); + SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRCTL, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRINFO1, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRINFO2, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRSAVE, v); + SET_SW_GCSR(csr, id, KVM_CSR_CTAG, v); + SET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v); + SET_SW_GCSR(csr, id, KVM_CSR_DERA, v); + SET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG1, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG2, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG3, v); + + SET_SW_GCSR(csr, id, KVM_CSR_PGD, v); + SET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v); + + ret = -1; + switch (id) { + case KVM_CSR_ESTAT: + kvm_write_gcsr_estat(*v); + /* estat IP0~IP7 inject through guestexcept */ + kvm_write_csr_gintc(((*v) >> 2) & 0xff); + ret = 0; + break; + default: + if (force && (id < CSR_ALL_SIZE)) { + kvm_set_sw_gcsr(csr, id, *v); + ret = 0; + } + break; + } + + return ret; +} + +struct kvm_iocsr { + u32 start, end; + int (*get) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 *res); + int (*set) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 val); +}; + +static struct kvm_iocsr_entry *_kvm_find_iocsr(struct kvm *kvm, u32 addr) +{ + int i = 0; + + for (i = 0; i < IOCSR_MAX; i++) { + if (addr == kvm->arch.iocsr[i].addr) + return &kvm->arch.iocsr[i]; + } + + return NULL; +} + +static int kvm_iocsr_common_get(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + int r = EMULATE_FAIL; + struct kvm_iocsr_entry *entry; + + spin_lock(&vcpu->kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(vcpu->kvm, addr); + if (entry) { + r = EMULATE_DONE; + *res = entry->data; + } + spin_unlock(&vcpu->kvm->arch.iocsr_lock); + return r; +} + +static int kvm_iocsr_common_set(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 val) +{ + int r = EMULATE_FAIL; + struct kvm_iocsr_entry *entry; + + spin_lock(&vcpu->kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(vcpu->kvm, addr); + if (entry) { + r = EMULATE_DONE; + entry->data = val; + } + spin_unlock(&vcpu->kvm->arch.iocsr_lock); + return r; +} + +static int kvm_misc_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + return kvm_iocsr_common_set(run, vcpu, addr, val); +} + +static int kvm_ipi_get(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 *res) +{ + int ret; + + ++vcpu->stat.rdcsr_ipi_access_exits; + run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, res); + if (ret) { + run->mmio.is_write = 0; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 0; + return EMULATE_DO_MMIO; + } + return EMULATE_DONE; +} + +static int kvm_extioi_isr_get(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + int ret; + + run->mmio.phys_addr = EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, res); + if (ret) { + run->mmio.is_write = 0; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 0; + return EMULATE_FAIL; + } + + return EMULATE_DONE; +} + +static int kvm_ipi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + int ret; + + run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, &val); + if (ret < 0) { + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} + +static int kvm_extioi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + int ret; + + if ((addr & 0x1f00) == KVM_IOCSR_EXTIOI_ISR_BASE) { + run->mmio.phys_addr = EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff)); + } else { + run->mmio.phys_addr = EXTIOI_ADDR((addr & 0x1fff)); + } + + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, &val); + if (ret < 0) { + memcpy(run->mmio.data, &val, run->mmio.len); + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} + +static int kvm_nop_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + return EMULATE_DONE; +} + +/* we put these iocsrs with access frequency, from high to low */ +static struct kvm_iocsr kvm_iocsrs[] = { + /* extioi iocsr */ + {KVM_IOCSR_EXTIOI_EN_BASE, KVM_IOCSR_EXTIOI_EN_BASE + 0x100, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_NODEMAP_BASE, KVM_IOCSR_EXTIOI_NODEMAP_BASE+0x28, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_ROUTE_BASE, KVM_IOCSR_EXTIOI_ROUTE_BASE + 0x100, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_ISR_BASE, KVM_IOCSR_EXTIOI_ISR_BASE + 0x1c, + kvm_extioi_isr_get, kvm_extioi_set}, + + {KVM_IOCSR_IPI_STATUS, KVM_IOCSR_IPI_STATUS + 0x40, + kvm_ipi_get, kvm_ipi_set}, + {KVM_IOCSR_IPI_SEND, KVM_IOCSR_IPI_SEND + 0x1, + NULL, kvm_ipi_set}, + {KVM_IOCSR_MBUF_SEND, KVM_IOCSR_MBUF_SEND + 0x1, + NULL, kvm_ipi_set}, + + {KVM_IOCSR_FEATURES, KVM_IOCSR_FEATURES + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_VENDOR, KVM_IOCSR_VENDOR + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_CPUNAME, KVM_IOCSR_CPUNAME + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_NODECNT, KVM_IOCSR_NODECNT + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_MISC_FUNC, KVM_IOCSR_MISC_FUNC + 0x1, + kvm_iocsr_common_get, kvm_misc_set}, +}; + +static int _kvm_emu_iocsr_read(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + enum emulation_result er = EMULATE_FAIL; + int i = 0; + struct kvm_iocsr *iocsr = NULL; + + if (!irqchip_in_kernel(vcpu->kvm)) { + run->iocsr_io.len = run->mmio.len; + run->iocsr_io.phys_addr = addr; + run->iocsr_io.is_write = 0; + return EMULATE_DO_IOCSR; + } + for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) { + iocsr = &kvm_iocsrs[i]; + if (addr >= iocsr->start && addr < iocsr->end) { + if (iocsr->get) + er = iocsr->get(run, vcpu, addr, res); + } + } + + if (er != EMULATE_DONE) + kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr); + + return er; +} + +static int _kvm_emu_iocsr_write(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 val) +{ + enum emulation_result er = EMULATE_FAIL; + int i = 0; + struct kvm_iocsr *iocsr = NULL; + + if (!irqchip_in_kernel(vcpu->kvm)) { + run->iocsr_io.len = run->mmio.len; + memcpy(run->iocsr_io.data, &val, run->iocsr_io.len); + run->iocsr_io.phys_addr = addr; + run->iocsr_io.is_write = 1; + return EMULATE_DO_IOCSR; + } + for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) { + iocsr = &kvm_iocsrs[i]; + if (addr >= iocsr->start && addr < iocsr->end) { + if (iocsr->set) + er = iocsr->set(run, vcpu, addr, val); + } + } + if (er != EMULATE_DONE) + kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr); + + return er; +} + +/* all iocsr operation should in kvm, no mmio */ +int _kvm_emu_iocsr(larch_inst inst, + struct kvm_run *run, struct kvm_vcpu *vcpu) +{ + u32 rd, rj, opcode; + u32 val; + u64 res = 0; + int ret; + + /* + * Each IOCSR with different opcode + */ + rd = inst.reg2_format.rd; + rj = inst.reg2_format.rj; + opcode = inst.reg2_format.opcode; + val = vcpu->arch.gprs[rj]; + res = vcpu->arch.gprs[rd]; + /* LoongArch is Little endian */ + switch (opcode) { + case iocsrrdb_op: + run->mmio.len = 1; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u8) res; + break; + case iocsrrdh_op: + run->mmio.len = 2; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u16) res; + break; + case iocsrrdw_op: + run->mmio.len = 4; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u32) res; + break; + case iocsrrdd_op: + run->mmio.len = 8; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = res; + break; + case iocsrwrb_op: + run->mmio.len = 1; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u8)res); + break; + case iocsrwrh_op: + run->mmio.len = 2; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u16)res); + break; + case iocsrwrw_op: + run->mmio.len = 4; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u32)res); + break; + case iocsrwrd_op: + run->mmio.len = 8; + ret = _kvm_emu_iocsr_write(run, vcpu, val, res); + break; + default: + ret = EMULATE_FAIL; + break; + } + + if (ret == EMULATE_DO_IOCSR) { + vcpu->arch.io_gpr = rd; + } + + return ret; +} + +int _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; + enum emulation_result er = EMULATE_DONE; + + switch (run->iocsr_io.len) { + case 8: + *gpr = *(s64 *)run->iocsr_io.data; + break; + case 4: + *gpr = *(int *)run->iocsr_io.data; + break; + case 2: + *gpr = *(short *)run->iocsr_io.data; + break; + case 1: + *gpr = *(char *) run->iocsr_io.data; + break; + default: + kvm_err("Bad IOCSR length: %d,addr is 0x%lx", + run->iocsr_io.len, vcpu->arch.badv); + er = EMULATE_FAIL; + break; + } + + return er; +} + +int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp) +{ + struct kvm_iocsr_entry *entry, tmp; + int r = -EFAULT; + + if (copy_from_user(&tmp, argp, sizeof(tmp))) + goto out; + + spin_lock(&kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(kvm, tmp.addr); + if (entry != NULL) + tmp.data = entry->data; + spin_unlock(&kvm->arch.iocsr_lock); + + if (entry) + r = copy_to_user(argp, &tmp, sizeof(tmp)); + +out: + return r; +} + +int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp) +{ + struct kvm_iocsr_entry *entry, tmp; + int r = -EFAULT; + + if (copy_from_user(&tmp, argp, sizeof(tmp))) + goto out; + + spin_lock(&kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(kvm, tmp.addr); + if (entry != NULL) { + r = 0; + entry->data = tmp.data; + } + spin_unlock(&kvm->arch.iocsr_lock); + +out: + return r; +} + +static struct kvm_iocsr_entry iocsr_array[IOCSR_MAX] = { + {KVM_IOCSR_FEATURES, .data = KVM_IOCSRF_NODECNT|KVM_IOCSRF_MSI + |KVM_IOCSRF_EXTIOI|KVM_IOCSRF_CSRIPI|KVM_IOCSRF_VM}, + {KVM_IOCSR_VENDOR, .data = 0x6e6f73676e6f6f4c}, /* Loongson */ + {KVM_IOCSR_CPUNAME, .data = 0x303030354133}, /* 3A5000 */ + {KVM_IOCSR_NODECNT, .data = 0x4}, + {KVM_IOCSR_MISC_FUNC, .data = 0x0}, +}; + +int _kvm_init_iocsr(struct kvm *kvm) +{ + int i = 0; + + spin_lock_init(&kvm->arch.iocsr_lock); + for (i = 0; i < IOCSR_MAX; i++) { + kvm->arch.iocsr[i].addr = iocsr_array[i].addr; + kvm->arch.iocsr[i].data = iocsr_array[i].data; + } + return 0; +} diff --git a/arch/loongarch/kvm/emulate.c b/arch/loongarch/kvm/emulate.c new file mode 100644 index 0000000000000000000000000000000000000000..43ba659ef03cb5bfe34010797bbdf06cf144c586 --- /dev/null +++ b/arch/loongarch/kvm/emulate.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include "trace.h" + +int _kvm_emu_idle(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.idle_exits; + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_IDLE); + if (!vcpu->arch.irq_pending) { + kvm_save_timer(vcpu); + kvm_vcpu_block(vcpu); + + /* + * We we are runnable, then definitely go off to user space to + * check if any I/O interrupts are pending. + */ + if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) { + kvm_clear_request(KVM_REQ_UNHALT, vcpu); + vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; + } + } + + if (kvm_check_request(KVM_REQ_EVENT, vcpu)) { + vcpu->arch.pv.pv_unhalted = false; + } + + return EMULATE_DONE; +} + +int _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) +{ + struct kvm_run *run = vcpu->run; + unsigned int rd, op8, opcode; + unsigned long rd_val = 0; + void *data = run->mmio.data; + unsigned long curr_pc; + int ret = 0; + + /* + * Update PC and hold onto current PC in case there is + * an error and we want to rollback the PC + */ + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + op8 = (inst.word >> 24) & 0xff; + run->mmio.phys_addr = vcpu->arch.badv; + if (run->mmio.phys_addr == KVM_INVALID_ADDR) + goto out_fail; + + if (op8 < 0x28) { + /* stptrw/d process */ + rd = inst.reg2i14_format.rd; + opcode = inst.reg2i14_format.opcode; + + switch (opcode) { + case stptrd_op: + run->mmio.len = 8; + *(unsigned long *)data = vcpu->arch.gprs[rd]; + break; + case stptrw_op: + run->mmio.len = 4; + *(unsigned int *)data = vcpu->arch.gprs[rd]; + break; + default: + break; + } + } else if (op8 < 0x30) { + /* st.b/h/w/d process */ + rd = inst.reg2i12_format.rd; + opcode = inst.reg2i12_format.opcode; + rd_val = vcpu->arch.gprs[rd]; + + switch (opcode) { + case std_op: + run->mmio.len = 8; + *(unsigned long *)data = rd_val; + break; + case stw_op: + run->mmio.len = 4; + *(unsigned int *)data = rd_val; + break; + case sth_op: + run->mmio.len = 2; + *(unsigned short *)data = rd_val; + break; + case stb_op: + run->mmio.len = 1; + *(unsigned char *)data = rd_val; + break; + default: + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + } else if (op8 == 0x38) { + /* stxb/h/w/d process */ + rd = inst.reg3_format.rd; + opcode = inst.reg3_format.opcode; + + switch (opcode) { + case stxb_op: + run->mmio.len = 1; + *(unsigned char *)data = vcpu->arch.gprs[rd]; + break; + case stxh_op: + run->mmio.len = 2; + *(unsigned short *)data = vcpu->arch.gprs[rd]; + break; + case stxw_op: + run->mmio.len = 4; + *(unsigned int *)data = vcpu->arch.gprs[rd]; + break; + case stxd_op: + run->mmio.len = 8; + *(unsigned long *)data = vcpu->arch.gprs[rd]; + break; + default: + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + } else { + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + + /* All MMIO emulate in kernel go through the common interface */ + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, data); + if (!ret) { + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + + return EMULATE_DO_MMIO; + +out_fail: + /* Rollback PC if emulation was unsuccessful */ + vcpu->arch.pc = curr_pc; + return EMULATE_FAIL; +} + + +int _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) +{ + unsigned int op8, opcode, rd; + int ret = 0; + struct kvm_run *run = vcpu->run; + + run->mmio.phys_addr = vcpu->arch.badv; + if (run->mmio.phys_addr == KVM_INVALID_ADDR) + return EMULATE_FAIL; + + vcpu->mmio_needed = 2; /* signed */ + op8 = (inst.word >> 24) & 0xff; + + if (op8 < 0x28) { + /* ldptr.w/d process */ + rd = inst.reg2i14_format.rd; + opcode = inst.reg2i14_format.opcode; + + switch (opcode) { + case ldptrd_op: + run->mmio.len = 8; + break; + case ldptrw_op: + run->mmio.len = 4; + break; + default: + break; + } + } else if (op8 < 0x2f) { + /* ld.b/h/w/d, ld.bu/hu/wu process */ + rd = inst.reg2i12_format.rd; + opcode = inst.reg2i12_format.opcode; + + switch (opcode) { + case ldd_op: + run->mmio.len = 8; + break; + case ldwu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 4; + break; + case ldw_op: + run->mmio.len = 4; + break; + case ldhu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 2; + break; + case ldh_op: + run->mmio.len = 2; + break; + case ldbu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 1; + break; + case ldb_op: + run->mmio.len = 1; + break; + default: + kvm_err("Load not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + } else if (op8 == 0x38) { + /* ldxb/h/w/d, ldxb/h/wu, ldgtb/h/w/d, ldleb/h/w/d process */ + rd = inst.reg3_format.rd; + opcode = inst.reg3_format.opcode; + + switch (opcode) { + case ldxb_op: + run->mmio.len = 1; + break; + case ldxbu_op: + run->mmio.len = 1; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxh_op: + run->mmio.len = 2; + break; + case ldxhu_op: + run->mmio.len = 2; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxw_op: + run->mmio.len = 4; + break; + case ldxwu_op: + run->mmio.len = 4; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxd_op: + run->mmio.len = 8; + break; + default: + kvm_err("Load not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + } else { + kvm_err("Load not yet supporded (inst=0x%08x) @ %lx\n", + inst.word, vcpu->arch.pc); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + + /* Set for _kvm_complete_mmio_read use */ + vcpu->arch.io_gpr = rd; + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, run->mmio.data); + run->mmio.is_write = 0; + vcpu->mmio_is_write = 0; + + if (!ret) { + _kvm_complete_mmio_read(vcpu, run); + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + return EMULATE_DO_MMIO; +} + +int _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; + enum emulation_result er = EMULATE_DONE; + + /* update with new PC */ + update_pc(&vcpu->arch); + switch (run->mmio.len) { + case 8: + *gpr = *(s64 *)run->mmio.data; + break; + + case 4: + if (vcpu->mmio_needed == 2) { + *gpr = *(int *)run->mmio.data; + } else + *gpr = *(unsigned int *)run->mmio.data; + break; + + case 2: + if (vcpu->mmio_needed == 2) + *gpr = *(short *) run->mmio.data; + else + *gpr = *(unsigned short *)run->mmio.data; + + break; + case 1: + if (vcpu->mmio_needed == 2) + *gpr = *(char *) run->mmio.data; + else + *gpr = *(unsigned char *) run->mmio.data; + break; + default: + kvm_err("Bad MMIO length: %d,addr is 0x%lx", + run->mmio.len, vcpu->arch.badv); + er = EMULATE_FAIL; + break; + } + + return er; +} diff --git a/arch/loongarch/kvm/entry.S b/arch/loongarch/kvm/entry.S new file mode 100644 index 0000000000000000000000000000000000000000..4bd6d9f213bea8968f0a231774a65434a8f6917e --- /dev/null +++ b/arch/loongarch/kvm/entry.S @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include "kvm_compat.h" + +#define RESUME_HOST (1 << 1) + +#define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x) +#define PT_GPR_OFFSET(x) (PT_R0 + 8*x) + + .text + +.macro kvm_save_guest_gprs base + .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_S $r\n, \base, GGPR_OFFSET(\n) + .endr +.endm + +.macro kvm_restore_guest_gprs base + .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_L $r\n, \base, GGPR_OFFSET(\n) + .endr +.endm + +.macro kvm_save_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_S $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +.macro kvm_restore_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_L $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +/* + * prepare switch to guest + * @param: + * KVM_ARCH: kvm_vcpu_arch, don't touch it until 'ertn' + * GPRNUM: KVM_ARCH gpr number + * tmp, tmp1: temp register + */ +.macro kvm_switch_to_guest KVM_ARCH GPRNUM tmp tmp1 + /* set host excfg.VS=0, all exceptions share one exception entry */ + csrrd \tmp, KVM_CSR_ECFG + bstrins.w \tmp, zero, (KVM_ECFG_VS_SHIFT + KVM_ECFG_VS_WIDTH - 1), KVM_ECFG_VS_SHIFT + csrwr \tmp, KVM_CSR_ECFG + + /* Load up the new EENTRY */ + KVM_LONG_L \tmp, \KVM_ARCH, KVM_ARCH_GEENTRY + csrwr \tmp, KVM_CSR_EENTRY + + /* Set Guest ERA */ + KVM_LONG_L \tmp, \KVM_ARCH, KVM_ARCH_GPC + csrwr \tmp, KVM_CSR_ERA + + /* Save host PGDL */ + csrrd \tmp, KVM_CSR_PGDL + KVM_LONG_S \tmp, \KVM_ARCH, KVM_ARCH_HPGD + + /* Switch to kvm */ + KVM_LONG_L \tmp1, \KVM_ARCH, KVM_VCPU_KVM - KVM_VCPU_ARCH + + /* Load guest PGDL */ + lu12i.w \tmp, KVM_GPGD + srli.w \tmp, \tmp, 12 + ldx.d \tmp, \tmp1, \tmp + csrwr \tmp, KVM_CSR_PGDL + + /* Mix GID and RID */ + csrrd \tmp1, KVM_CSR_GSTAT + bstrpick.w \tmp1, \tmp1, (KVM_GSTAT_GID_SHIFT + KVM_GSTAT_GID_WIDTH - 1), KVM_GSTAT_GID_SHIFT + csrrd \tmp, KVM_CSR_GTLBC + bstrins.w \tmp, \tmp1, (KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1), KVM_GTLBC_TGID_SHIFT + csrwr \tmp, KVM_CSR_GTLBC + + /* + * Switch to guest: + * GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 + * ertn + */ + + /* + * Enable intr in root mode with future ertn so that host interrupt + * can be responsed during VM runs + * guest crmd comes from separate gcsr_CRMD register + */ + ori \tmp, zero, KVM_PRMD_PIE + csrxchg \tmp, \tmp, KVM_CSR_PRMD + + /* Set PVM bit to setup ertn to guest context */ + ori \tmp, zero, KVM_GSTAT_PVM + csrxchg \tmp, \tmp, KVM_CSR_GSTAT + + /* Load Guest gprs */ + kvm_restore_guest_gprs \KVM_ARCH + + /* Load KVM_ARCH register */ + KVM_LONG_L \KVM_ARCH, \KVM_ARCH, GGPR_OFFSET(\GPRNUM) + + ertn +.endm + +#ifndef EXCPTION_ENTRY +#define EXCPTION_ENTRY(name) \ + .globl name ASM_NL \ + .p2align 12; \ + name: \ + .cfi_startproc; +#endif +#ifndef EXCPTION_ENDPROC +#define EXCPTION_ENDPROC(name) \ + .cfi_endproc; \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* load kvm_vcpu to a2 and store a1 for free use */ +EXCPTION_ENTRY(kvm_exception_entry) + csrwr a2, KVM_TEMP_KS + csrrd a2, KVM_VCPU_KS + KVM_LONG_ADDI a2, a2, KVM_VCPU_ARCH + + /* After save gprs, free to use any gpr */ + kvm_save_guest_gprs a2 + /* Save guest a2 */ + csrrd t0, KVM_TEMP_KS + KVM_LONG_S t0, a2, GGPR_OFFSET(REG_A2) + + b kvm_exit_entry +EXCPTION_ENDPROC(kvm_exception_entry) + +/* a2: kvm_vcpu_arch, a1 is free to use */ +SYM_FUNC_START(kvm_exit_entry) + csrrd s1, KVM_VCPU_KS + KVM_LONG_L s0, s1, KVM_VCPU_RUN + + csrrd t0, KVM_CSR_ESTAT + KVM_LONG_S t0, a2, KVM_ARCH_HESTAT + csrrd t0, KVM_CSR_ERA + KVM_LONG_S t0, a2, KVM_ARCH_GPC + csrrd t0, KVM_CSR_BADV + KVM_LONG_S t0, a2, KVM_ARCH_HBADV + csrrd t0, KVM_CSR_BADI + KVM_LONG_S t0, a2, KVM_ARCH_HBADI + + /* Restore host excfg.VS */ + csrrd t0, KVM_CSR_ECFG + KVM_LONG_L t1, a2, KVM_ARCH_HECFG + or t0, t0, t1 + csrwr t0, KVM_CSR_ECFG + + /* Restore host eentry */ + KVM_LONG_L t0, a2, KVM_ARCH_HEENTRY + csrwr t0, KVM_CSR_EENTRY + + /* restore host pgd table */ + KVM_LONG_L t0, a2, KVM_ARCH_HPGD + csrwr t0, KVM_CSR_PGDL + + /* + * Disable PGM bit to enter root mode by default with next ertn + */ + ori t0, zero, KVM_GSTAT_PVM + csrxchg zero, t0, KVM_CSR_GSTAT + + /* + * Clear GTLBC.TGID field + * 0: for root tlb update in future tlb instr + * others: for guest tlb update like gpa to hpa in future tlb instr + */ + csrrd t0, KVM_CSR_GTLBC + bstrins.w t0, zero, KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1, KVM_GTLBC_TGID_SHIFT + csrwr t0, KVM_CSR_GTLBC + + KVM_LONG_L tp, a2, KVM_ARCH_HGP + KVM_LONG_L sp, a2, KVM_ARCH_HSTACK + /* Restore per cpu base register */ + KVM_LONG_L $r21, a2, KVM_ARCH_HPERCPU + + KVM_LONG_ADDI sp, sp, -PT_SIZE + + /* Prepare handle exception */ + or a0, s0, zero + or a1, s1, zero + KVM_LONG_L t8, a2, KVM_ARCH_HANDLE_EXIT + jirl ra,t8, 0 + + or a2, s1, zero + KVM_LONG_ADDI a2, a2, KVM_VCPU_ARCH + + andi t0, a0, RESUME_HOST + bnez t0, ret_to_host + INT_S zero, a2, KVM_ARCH_ISHYPCALL + +ret_to_guest: + /* Save per cpu register again, maybe switched to another cpu */ + KVM_LONG_S $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr s1, KVM_VCPU_KS + kvm_switch_to_guest a2 REG_A2 t0 t1 + +ret_to_host: + KVM_LONG_L a2, a2, KVM_ARCH_HSTACK + addi.d a2, a2, -PT_SIZE + srai.w a3, a0, 2 + or a0, a3, zero + kvm_restore_host_gpr a2 + jirl zero, ra, 0 +SYM_FUNC_END(kvm_exit_entry) + +/* + * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu) + * + * @register_param: + * a0: kvm_run* run + * a1: kvm_vcpu* vcpu + */ +SYM_FUNC_START(kvm_enter_guest) + /* allocate space in stack bottom */ + KVM_LONG_ADDI a2, sp, -PT_SIZE + + /* save host gprs */ + kvm_save_host_gpr a2 + + /* save host crmd,prmd csr to stack */ + csrrd a3, KVM_CSR_CRMD + KVM_LONG_S a3, a2, PT_CRMD + csrrd a3, KVM_CSR_PRMD + KVM_LONG_S a3, a2, PT_PRMD + + KVM_LONG_ADDI a2, a1, KVM_VCPU_ARCH + KVM_LONG_S sp, a2, KVM_ARCH_HSTACK + KVM_LONG_S tp, a2, KVM_ARCH_HGP + /* Save per cpu base register */ + KVM_LONG_S $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr a1, KVM_VCPU_KS + + kvm_switch_to_guest a2 REG_A2 t0 t1 + +SYM_FUNC_END(kvm_enter_guest) + +SYM_FUNC_START(__kvm_save_fpu) + fpu_save_csr a0 t1 + fpu_save_double a0 t1 + fpu_save_cc a0 t1 t2 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_fpu) + +SYM_FUNC_START(__kvm_restore_fpu) + fpu_restore_double a0 t1 # clobbers t1 + fpu_restore_cc a0 t1 t2 # clobbers t1, t2 + fpu_restore_csr a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_fpu) + +#ifdef CONFIG_CPU_HAS_LSX +SYM_FUNC_START(__kvm_save_lsx) + fpu_save_csr a0 t1 + fpu_save_cc a0 t1 t2 + lsx_save_data a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_lsx) + +SYM_FUNC_START(__kvm_restore_lsx) + lsx_restore_data a0 t1 + fpu_restore_cc a0 t1 t2 # clobbers t1, t2 + fpu_restore_csr a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lsx) + +SYM_FUNC_START(__kvm_restore_lsx_upper) + lsx_restore_all_upper a0 t0 t1 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lsx_upper) +#endif + +#ifdef CONFIG_CPU_HAS_LASX +SYM_FUNC_START(__kvm_save_lasx) + fpu_save_csr a0 t1 + fpu_save_cc a0 t1 t2 + lasx_save_data a0 t1 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_lasx) + +SYM_FUNC_START(__kvm_restore_lasx) + lasx_restore_data a0 t1 + fpu_restore_cc a0 t1 t2 # clobbers t1, t2 + fpu_restore_csr a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lasx) +#endif diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c new file mode 100644 index 0000000000000000000000000000000000000000..9e2220a64ed3d8d898410a8b1160bd314be45e7d --- /dev/null +++ b/arch/loongarch/kvm/exit.c @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include + +#include "trace.h" +#include "kvm_compat.h" +#include "kvmcsr.h" +#include "intc/ls3a_ext_irq.h" + +/* + * Loongarch KVM callback handling for not implemented guest exiting + */ +static int _kvm_fault_ni(struct kvm_vcpu *vcpu) +{ + unsigned long estat, badv; + unsigned int exccode, inst; + + /* + * Fetch the instruction. + */ + badv = vcpu->arch.badv; + estat = vcpu->arch.host_estat; + exccode = (estat & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT; + inst = vcpu->arch.badi; + kvm_err("Exccode: %d PC=%#lx inst=0x%08x BadVaddr=%#lx estat=%#llx\n", + exccode, vcpu->arch.pc, inst, badv, kvm_read_gcsr_estat()); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; +} + +static int _kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) +{ + enum emulation_result er = EMULATE_DONE; + unsigned int rd, rj, csrid; + unsigned long csr_mask; + unsigned long val = 0; + + /* + * CSR value mask imm + * rj = 0 means csrrd + * rj = 1 means csrwr + * rj != 0,1 means csrxchg + */ + rd = inst.reg2csr_format.rd; + rj = inst.reg2csr_format.rj; + csrid = inst.reg2csr_format.csr; + + /* Process CSR ops */ + if (rj == 0) { + /* process csrrd */ + val = _kvm_emu_read_csr(vcpu, csrid); + if (er != EMULATE_FAIL) + vcpu->arch.gprs[rd] = val; + } else if (rj == 1) { + /* process csrwr */ + val = vcpu->arch.gprs[rd]; + _kvm_emu_write_csr(vcpu, csrid, val); + } else { + /* process csrxchg */ + val = vcpu->arch.gprs[rd]; + csr_mask = vcpu->arch.gprs[rj]; + _kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val); + } + + return er; +} + +static int _kvm_emu_cache(struct kvm_vcpu *vcpu, larch_inst inst) +{ + return EMULATE_DONE; +} + +static int _kvm_trap_handle_gspr(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + struct kvm_run *run = vcpu->run; + larch_inst inst; + unsigned long curr_pc; + int rd, rj; + unsigned int index; + + /* + * Fetch the instruction. + */ + inst.word = vcpu->arch.badi; + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + er = EMULATE_FAIL; + switch (((inst.word >> 24) & 0xff)) { + case 0x0: + /* cpucfg GSPR */ + if (inst.reg2_format.opcode == 0x1B) { + rd = inst.reg2_format.rd; + rj = inst.reg2_format.rj; + ++vcpu->stat.cpucfg_exits; + index = vcpu->arch.gprs[rj]; + vcpu->arch.gprs[rd] = vcpu->kvm->arch.cpucfgs.cpucfg[index]; + if ((index == 2) || (vcpu->arch.gprs[rd] == 0)) + /* + * Fallback to get host cpucfg info, this is just for + * compatible with older qemu. + */ + vcpu->arch.gprs[rd] = read_cpucfg(index); + if (index == 2) + /* do not support nested virtualization */ + vcpu->arch.gprs[rd] &= ~CPUCFG2_LVZP; + er = EMULATE_DONE; + } + break; + case 0x4: + /* csr GSPR */ + er = _kvm_handle_csr(vcpu, inst); + break; + case 0x6: + /* iocsr,cacop,idle GSPR */ + switch (((inst.word >> 22) & 0x3ff)) { + case 0x18: + /* cache GSPR */ + er = _kvm_emu_cache(vcpu, inst); + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); + break; + case 0x19: + /* iocsr/idle GSPR */ + switch (((inst.word >> 15) & 0x1ffff)) { + case 0xc90: + /* iocsr GSPR */ + er = _kvm_emu_iocsr(inst, run, vcpu); + break; + case idle_op: + /* idle GSPR */ + er = _kvm_emu_idle(vcpu); + break; + default: + er = EMULATE_FAIL; + break; + } + break; + default: + er = EMULATE_FAIL; + break; + } + break; + default: + er = EMULATE_FAIL; + break; + } + + /* Rollback PC only if emulation was unsuccessful */ + if (er == EMULATE_FAIL) { + kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n", + curr_pc, __func__, inst.word); + + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->arch.pc = curr_pc; + } + return er; +} + +static int _kvm_check_hypcall(struct kvm_vcpu *vcpu) +{ + enum emulation_result ret; + larch_inst inst; + unsigned long curr_pc; + unsigned int code; + + /* + * Update PC and hold onto current PC in case there is + * an error and we want to rollback the PC + */ + inst.word = vcpu->arch.badi; + code = inst.reg0i15_format.simmediate; + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + ret = EMULATE_DONE; + switch (code) { + case KVM_HC_CODE_SERIVCE: + ret = EMULATE_PV_HYPERCALL; + break; + case KVM_HC_CODE_SWDBG: + /* + * Only SWDBG(SoftWare DeBug) could stop vm + * code other than 0 is ignored. + */ + ret = EMULATE_DEBUG; + break; + default: + kvm_info("[%#lx] HYPCALL %#03x unsupported\n", vcpu->arch.pc, code); + break; + } + + if (ret == EMULATE_DEBUG) + vcpu->arch.pc = curr_pc; + + return ret; +} + +/* Execute cpucfg instruction will tirggerGSPR, + * Also the access to unimplemented csrs 0x15 + * 0x16, 0x50~0x53, 0x80, 0x81, 0x90~0x95, 0x98 + * 0xc0~0xff, 0x100~0x109, 0x500~0x502, + * cacop_op, idle_op iocsr ops the same */ +static int _kvm_handle_gspr(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + vcpu->arch.is_hypcall = 0; + + er = _kvm_trap_handle_gspr(vcpu); + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + vcpu->run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else if (er == EMULATE_DO_IOCSR) { + vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR; + ret = RESUME_HOST; + } else { + kvm_err("%s internal error\n", __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_hypcall(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + vcpu->arch.is_hypcall = 0; + er = _kvm_check_hypcall(vcpu); + + if (er == EMULATE_PV_HYPERCALL) + ret = _kvm_handle_pv_hcall(vcpu); + else if (er == EMULATE_DEBUG) { + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + ret = RESUME_HOST; + } else + ret = RESUME_GUEST; + + return ret; +} + +static int _kvm_handle_gcm(struct kvm_vcpu *vcpu) +{ + int ret, subcode; + + vcpu->arch.is_hypcall = 0; + ret = RESUME_GUEST; + subcode = (vcpu->arch.host_estat & KVM_ESTAT_ESUBCODE) >> KVM_ESTAT_ESUBCODE_SHIFT; + if ((subcode != EXCSUBCODE_GCSC) && (subcode != EXCSUBCODE_GCHC)) { + kvm_err("%s internal error\n", __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + + return ret; +} + +/** + * _kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use fpu which hasn't been allowed + * by the root context. + */ +static int _kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If guest FPU not present, the FPU operation should have been + * treated as a reserved instruction! + * If FPU already in use, we shouldn't get this at all. + */ + if (WARN_ON(!_kvm_guest_has_fpu(&vcpu->arch) || + vcpu->arch.aux_inuse & KVM_LARCH_FPU)) { + kvm_err("%s internal error\n", __func__); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + kvm_own_fpu(vcpu); + return RESUME_GUEST; +} + +/** + * _kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LSX when it is disabled in the root + * context. + */ +static int _kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If LSX not present or not exposed to guest, the LSX operation + * should have been treated as a reserved instruction! + * If LSX already in use, we shouldn't get this at all. + */ + if (!_kvm_guest_has_lsx(&vcpu->arch) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) || + vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + kvm_err("%s internal error, lsx %d guest euen %llx aux %x", + __func__, _kvm_guest_has_lsx(&vcpu->arch), + kvm_read_gcsr_euen(), vcpu->arch.aux_inuse); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + kvm_own_lsx(vcpu); + return RESUME_GUEST; +} + +bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu) +{ + return cpu_has_lasx && vcpu->arch.lsx_enabled && vcpu->kvm->arch.cpucfg_lasx; +} + +/** + * _kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LASX when it is disabled in the root + * context. + */ +static int _kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If LASX not present or not exposed to guest, the LASX operation + * should have been treated as a reserved instruction! + * If LASX already in use, we shouldn't get this at all. + */ + if (!_kvm_guest_has_lasx(vcpu) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LASXEN) || + vcpu->arch.aux_inuse & KVM_LARCH_LASX) { + kvm_err("%s internal error, lasx %d guest euen %llx aux %x", + __func__, _kvm_guest_has_lasx(vcpu), + kvm_read_gcsr_euen(), vcpu->arch.aux_inuse); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + kvm_own_lasx(vcpu); + + return RESUME_GUEST; +} + +/** + * _kvm_handle_fpu_disabled() - Guest used lbt however it is disabled at host + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use lbt which hasn't been allowed + * by the root context. + */ +static int _kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If guest LBT not present, the LBT operation should have been + * treated as a reserved instruction! + * If LBT already in use, we shouldn't get this at all. + */ + if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) { + kvm_err("%s internal error\n", __func__); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + kvm_own_lbt(vcpu); + return RESUME_GUEST; +} + +static int _kvm_handle_read_fault(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + ulong badv = vcpu->arch.badv; + larch_inst inst; + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + if (kvm_handle_mm_fault(vcpu, badv, false)) { + /* A code fetch fault doesn't count as an MMIO */ + if (kvm_is_ifetch_fault(&vcpu->arch)) { + kvm_err("%s ifetch error addr:%lx\n", __func__, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + /* Treat as MMIO */ + inst.word = vcpu->arch.badi; + er = _kvm_emu_mmio_read(vcpu, inst); + if (er == EMULATE_FAIL) { + kvm_err("Guest Emulate Load failed: PC: %#lx, BadVaddr: %#lx\n", + vcpu->arch.pc, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + } + } + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else { + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_write_fault(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + ulong badv = vcpu->arch.badv; + larch_inst inst; + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + if (kvm_handle_mm_fault(vcpu, badv, true)) { + + /* Treat as MMIO */ + inst.word = vcpu->arch.badi; + er = _kvm_emu_mmio_write(vcpu, inst); + if (er == EMULATE_FAIL) { + kvm_err("Guest Emulate Store failed: PC: %#lx, BadVaddr: %#lx\n", + vcpu->arch.pc, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + } + } + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else { + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_debug(struct kvm_vcpu *vcpu) +{ + uint32_t fwps, mwps; + + fwps = kvm_csr_readq(KVM_CSR_FWPS); + mwps = kvm_csr_readq(KVM_CSR_MWPS); + if (fwps & 0xff) + kvm_csr_writeq(fwps, KVM_CSR_FWPS); + if (mwps & 0xff) + kvm_csr_writeq(mwps, KVM_CSR_MWPS); + vcpu->run->debug.arch.exception = EXCCODE_WATCH; + vcpu->run->debug.arch.fwps = fwps; + vcpu->run->debug.arch.mwps = mwps; + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + return RESUME_HOST; +} + +static exit_handle_fn _kvm_fault_tables[EXCCODE_INT_START] = { + [EXCCODE_TLBL] = _kvm_handle_read_fault, + [EXCCODE_TLBS] = _kvm_handle_write_fault, + [EXCCODE_TLBI] = _kvm_handle_read_fault, + [EXCCODE_TLBM] = _kvm_handle_write_fault, + [EXCCODE_TLBNR] = _kvm_handle_read_fault, + [EXCCODE_TLBNX] = _kvm_handle_read_fault, + [EXCCODE_FPDIS] = _kvm_handle_fpu_disabled, + [EXCCODE_LSXDIS] = _kvm_handle_lsx_disabled, + [EXCCODE_LASXDIS] = _kvm_handle_lasx_disabled, + [EXCCODE_WATCH] = _kvm_handle_debug, + [EXCCODE_GSPR] = _kvm_handle_gspr, + [EXCCODE_HVC] = _kvm_handle_hypcall, + [EXCCODE_GCM] = _kvm_handle_gcm, + [EXCCODE_BTDIS] = _kvm_handle_lbt_disabled, +}; + +int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) +{ + return _kvm_fault_tables[fault](vcpu); +} + +void _kvm_init_fault(void) +{ + int i; + + for (i = 0; i < EXCCODE_INT_START; i++) + if (!_kvm_fault_tables[i]) + _kvm_fault_tables[i] = _kvm_fault_ni; +} diff --git a/arch/loongarch/kvm/fpu.c b/arch/loongarch/kvm/fpu.c new file mode 100644 index 0000000000000000000000000000000000000000..56a7e1ba70b7b1f94263242dd3fcf760b8c0de88 --- /dev/null +++ b/arch/loongarch/kvm/fpu.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include + +/* FPU/LSX context management */ +void __kvm_save_fpu(struct loongarch_fpu *fpu); +void __kvm_restore_fpu(struct loongarch_fpu *fpu); + +void kvm_save_fpu(struct kvm_vcpu *cpu) +{ + return __kvm_save_fpu(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_fpu); + +void kvm_restore_fpu(struct kvm_vcpu *cpu) +{ + return __kvm_restore_fpu(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_fpu); + +#ifdef CONFIG_CPU_HAS_LSX +void __kvm_save_lsx(struct loongarch_fpu *fpu); +void __kvm_restore_lsx(struct loongarch_fpu *fpu); +void __kvm_restore_lsx_upper(struct loongarch_fpu *fpu); + +void kvm_save_lsx(struct kvm_vcpu *cpu) +{ + return __kvm_save_lsx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_lsx); + +void kvm_restore_lsx(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lsx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lsx); + +void kvm_restore_lsx_upper(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lsx_upper(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lsx_upper); + +#endif + +#ifdef CONFIG_CPU_HAS_LSX +void __kvm_save_lasx(struct loongarch_fpu *fpu); +void __kvm_restore_lasx(struct loongarch_fpu *fpu); +void __kvm_restore_lasx_upper(struct loongarch_fpu *fpu); + +void kvm_save_lasx(struct kvm_vcpu *cpu) +{ + return __kvm_save_lasx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_lasx); + +void kvm_restore_lasx(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lasx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lasx); + +void kvm_restore_lasx_upper(struct kvm_vcpu *cpu) +{ + return _restore_lasx_upper(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lasx_upper); +#endif + +#ifdef CONFIG_CPU_HAS_LBT +void kvm_restore_lbt(struct kvm_vcpu *cpu) +{ + restore_lbt_registers(&cpu->arch.lbt); +} +EXPORT_SYMBOL_GPL(kvm_restore_lbt); + +void kvm_save_lbt(struct kvm_vcpu *cpu) +{ + save_lbt_registers(&cpu->arch.lbt); +} +EXPORT_SYMBOL_GPL(kvm_save_lbt); +#endif + + +EXPORT_SYMBOL_GPL(kvm_enter_guest); +EXPORT_SYMBOL_GPL(kvm_exception_entry); + diff --git a/arch/loongarch/kvm/hypcall.c b/arch/loongarch/kvm/hypcall.c new file mode 100644 index 0000000000000000000000000000000000000000..83b3ef0e8a839788d5bcdc67a157cedfb7259d16 --- /dev/null +++ b/arch/loongarch/kvm/hypcall.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include "intc/ls3a_ipi.h" + +int kvm_virt_ipi(struct kvm_vcpu *vcpu) +{ + int ret = 0; + u64 ipi_bitmap; + unsigned int min, action, cpu; + + ipi_bitmap = vcpu->arch.gprs[REG_A1]; + min = vcpu->arch.gprs[REG_A2]; + action = vcpu->arch.gprs[REG_A3]; + + if (ipi_bitmap) { + cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG); + while (cpu < BITS_PER_LONG) { + kvm_helper_send_ipi(vcpu, cpu + min, action); + cpu = find_next_bit((void *)&ipi_bitmap, BITS_PER_LONG, cpu + 1); + } + } + + return ret; +} + +int kvm_save_notify(struct kvm_vcpu *vcpu) +{ + unsigned long num, id, data; + struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache; + + int ret = 0; + + num = vcpu->arch.gprs[REG_A0]; + id = vcpu->arch.gprs[REG_A1]; + data = vcpu->arch.gprs[REG_A2]; + + switch (id) { + case KVM_FEATURE_STEAL_TIME: + if (!sched_info_on()) + break; + + if (vcpu->arch.st.guest_addr && (data == 0)) + kvm_release_pfn(cache->pfn, cache->dirty, cache); + + vcpu->arch.st.guest_addr = data; + kvm_debug("cpu :%d addr:%lx\n", vcpu->vcpu_id, data); + vcpu->arch.st.last_steal = current->sched_info.run_delay; + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); + break; + default: + break; + }; + + return ret; +}; + +static int _kvm_pv_feature(struct kvm_vcpu *vcpu) +{ + int feature = vcpu->arch.gprs[REG_A1]; + int ret = KVM_RET_NOT_SUPPORTED; + switch (feature) { + case KVM_FEATURE_STEAL_TIME: + if (sched_info_on()) + ret = KVM_RET_SUC; + break; + case KVM_FEATURE_MULTI_IPI: + ret = KVM_RET_SUC; + break; + case KVM_FEATURE_PARAVIRT_SPINLOCK: +#ifdef CONFIG_PARAVIRT_SPINLOCKS + ret = KVM_RET_SUC; +#endif + break; + default: + break; + } + return ret; +} + +static int kvm_pv_kick_cpu(struct kvm_vcpu *vcpu) +{ + int cpu = vcpu->arch.gprs[REG_A1]; + struct kvm_vcpu *dst_vcpu = vcpu->kvm->vcpus[cpu]; + int ret = 0; + + dst_vcpu->arch.pv.pv_unhalted = true; + kvm_make_request(KVM_REQ_EVENT, dst_vcpu); + kvm_vcpu_kick(dst_vcpu); + + return ret; +} + +/* + * hypcall emulation always return to guest, Caller should check retval. + */ +int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu) +{ + unsigned long func = vcpu->arch.gprs[REG_A0]; + int hyp_ret = KVM_RET_NOT_SUPPORTED; + + switch (func) { + case KVM_HC_FUNC_FEATURE: + hyp_ret = _kvm_pv_feature(vcpu); + break; + case KVM_HC_FUNC_NOTIFY: + hyp_ret = kvm_save_notify(vcpu); + break; + case KVM_HC_FUNC_IPI: + hyp_ret = kvm_virt_ipi(vcpu); + break; + case KVM_HC_KICK_CPU: + hyp_ret = kvm_pv_kick_cpu(vcpu); + break; + default: + kvm_info("[%#lx] hvc func:%#lx unsupported\n", vcpu->arch.pc, func); + break; + }; + + vcpu->arch.gprs[REG_A0] = hyp_ret; + + return RESUME_GUEST; +} diff --git a/arch/loongarch/kvm/intc/irqchip-debug.c b/arch/loongarch/kvm/intc/irqchip-debug.c new file mode 100644 index 0000000000000000000000000000000000000000..488b00366b47554603f43e1611516955e9a18526 --- /dev/null +++ b/arch/loongarch/kvm/intc/irqchip-debug.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include "kvmcpu.h" +#include "ls3a_ext_irq.h" +#include "ls7a_irq.h" + +#ifdef CONFIG_DEBUG_FS +static int irqchip_state_show(struct seq_file *m, void *v) +{ + struct kvm *kvm = m->private; + + kvm_get_kvm(kvm); + kvm_dump_ls3a_extirq_state(m, kvm->arch.v_extirq); + kvm_dump_ls7a_ioapic_state(m, kvm->arch.v_ioapic); + kvm_put_kvm(kvm); + + return 0; +} + +static int irqchip_debug_open(struct inode *inode, struct file *file) +{ + return single_open(file, irqchip_state_show, inode->i_private); +} + +static const struct file_operations irqchip_debug_fops = { + .open = irqchip_debug_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void irqchip_debug_init(struct kvm *kvm) +{ + debugfs_create_file("irqchip-state", 0444, kvm->debugfs_dentry, kvm, + &irqchip_debug_fops); +} +#else + +void irqchip_debug_init(struct kvm *kvm) {} +#endif +void irqchip_debug_destroy(struct kvm *kvm) +{ +} diff --git a/arch/loongarch/kvm/intc/ls3a_ext_irq.c b/arch/loongarch/kvm/intc/ls3a_ext_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..a110a0e7e1ce3610a0e30f00d024aaf1e4385278 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ext_irq.c @@ -0,0 +1,922 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include "kvmcpu.h" +#include "kvm_compat.h" +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +#define ls3a_ext_irq_lock(s, flags) spin_lock_irqsave(&s->lock, flags) +#define ls3a_ext_irq_unlock(s, flags) spin_unlock_irqrestore(&s->lock, flags) + +extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq); +void ext_deactive_core_isr(struct kvm *kvm, int irq_num, int vcpu_id) +{ + int ipnum; + unsigned long found1; + struct kvm_loongarch_interrupt irq; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + ipnum = state->ext_sw_ipmap[irq_num]; + + bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1); + bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + kvm_debug("vcpu_id %d irqnum %d found:0x%lx ipnum %d down\n", vcpu_id, irq_num, found1, ipnum); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = -(ipnum + 2); /* IP2~IP5 */ + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } +} + +/** + * ext_irq_update_core() + * @kvm: KVM structure pointer + * @irq_num: 0~256 ext irq num + * @level: 0~1 High and low level + * + * Route the status of the extended interrupt to the host CPU core. + * + */ +void ext_irq_update_core(struct kvm *kvm, int irq_num, int level) +{ + int nrcpus, ipnum, vcpu_id; + unsigned long found1; + struct kvm_loongarch_interrupt irq; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + nrcpus = atomic_read(&kvm->online_vcpus); + vcpu_id = state->ext_sw_coremap[irq_num]; + ipnum = state->ext_sw_ipmap[irq_num]; + + if (vcpu_id > (nrcpus - 1)) { + vcpu_id = 0; + } + + if (level == 1) { + if (test_bit(irq_num, (void *)state->ext_en.reg_u8) == false) { + return; + } + if (test_bit(irq_num, (void *)state->ext_isr.reg_u8) == false) { + return; + } + bitmap_set((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + bitmap_set((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + kvm_debug("%s:%d --- vcpu_id %d irqnum %d found1 0x%lx ipnum %d\n", + __FUNCTION__, __LINE__, vcpu_id, irq_num, found1, ipnum); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = ipnum + 2; /* IP2~IP5 */ + kvm_debug("%s:%d --- vcpu_id %d ipnum %d raise\n", + __FUNCTION__, __LINE__, vcpu_id, ipnum); + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } + } else { + bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1); + bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = -(ipnum + 2); /* IP2~IP5 */ + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } + + } +} + +void msi_irq_handler(struct kvm *kvm, int irq, int level) +{ + unsigned long flags; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + kvm_debug("ext_irq_handler:irq = %d,level = %d\n", irq, level); + + ls3a_ext_irq_lock(s, flags); + if (level == 1) { + if (test_bit(irq, (void *)&state->ext_isr)) + goto out; + __set_bit(irq, (void *)&state->ext_isr); + } else { + if (!test_bit(irq, (void *)&state->ext_isr)) + goto out; + __clear_bit(irq, (void *)&state->ext_isr); + } + + ext_irq_update_core(kvm, irq, level); +out: + ls3a_ext_irq_unlock(s, flags); +} + +static int ls3a_ext_intctl_readb(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START); + *(uint8_t *)val = state->ext_en.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START); + *(uint8_t *)val = state->bounce.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START); + *(uint8_t *)val = state->ext_isr.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~31 */ + vcpu_id = (offset >> 8) & 0xff; + reg_count = offset & 0xff; + *(uint8_t *)val = state->ext_core_isr.reg_u8[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + reg_count = (offset - EXTIOI_IPMAP_START); + *(uint8_t *)val = state->ip_map.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START); + *(uint8_t *)val = state->core_map.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START); + *(uint8_t *)val = state->node_type.reg_u8[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%x\n", + __FUNCTION__, addr, *(uint8_t *)val); + return 0; +} + +static int ls3a_ext_intctl_readw(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 4; + *(uint32_t *)val = state->ext_en.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 4; + *(uint32_t *)val = state->bounce.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START) / 4; + *(uint32_t *)val = state->ext_isr.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~7*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 4; + *(uint32_t *)val = state->ext_core_isr.reg_u32[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + reg_count = (offset - EXTIOI_IPMAP_START) / 4; + *(uint32_t *)val = state->ip_map.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START) / 4; + *(uint32_t *)val = state->core_map.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 4; + *(uint32_t *)val = state->node_type.reg_u32[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%x\n", + __FUNCTION__, addr, *(uint32_t *)val); + + return 0; +} + +static int ls3a_ext_intctl_readl(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 8; + *(uint64_t *)val = state->ext_en.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 8; + *(uint64_t *)val = state->bounce.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START) / 8; + *(uint64_t *)val = state->ext_isr.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~3*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 8; + + *(uint64_t *)val = state->ext_core_isr.reg_u64[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + *(uint64_t *)val = state->ip_map.reg_u64; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START) / 8; + *(uint64_t *)val = state->core_map.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 8; + *(uint64_t *)val = state->node_type.reg_u64[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%llx\n", + __FUNCTION__, addr, *(uint64_t *)val); + return 0; +} +/** + * ls3a_ext_intctl_read() + * @kvm: KVM structure pointer + * @addr: Register address + * @size: The width of the register to be read. + * @val: The pointer to the read result. + * + * Analog extended interrupt related register read. + * + */ +static int ls3a_ext_intctl_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int size, void *val) +{ + struct ls3a_kvm_extirq *s = NULL; + unsigned long flags; + uint64_t offset; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + offset = addr & 0xfffff; + if (offset & (size - 1)) { + printk("%s:unaligned address access %llx size %d\n", + __FUNCTION__, addr, size); + return 0; + } + addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF; + ls3a_ext_irq_lock(s, flags); + + switch (size) { + case 1: + ls3a_ext_intctl_readb(vcpu, dev, addr, val); + break; + case 4: + ls3a_ext_intctl_readw(vcpu, dev, addr, val); + break; + case 8: + ls3a_ext_intctl_readl(vcpu, dev, addr, val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx, size %d\n", + __FUNCTION__, addr, size); + } + ls3a_ext_irq_unlock(s, flags); + kvm_debug("%s(%d):address access %llx size %d\n", + __FUNCTION__, __LINE__, offset, size); + + return 0; +} + +static int ls3a_ext_intctl_writeb(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, reg_count; + uint8_t val_data_u8, old_data_u8; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + int mask, level, i, irqnum, ipnum; + int vcpu_id; + + unsigned long val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u8 = val & 0xffUL; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START); + old_data_u8 = state->ext_en.reg_u8[reg_count]; + if (old_data_u8 != val_data_u8) { + state->ext_en.reg_u8[reg_count] = val_data_u8; + old_data_u8 = old_data_u8 ^ val_data_u8; + mask = 0x1; + for (i = 0; i < 8; i++) { + if (old_data_u8 & mask) { + level = !!(val_data_u8 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 8, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START); + state->bounce.reg_u8[reg_count] = val_data_u8; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) & 0x1f; + old_data_u8 = state->ext_isr.reg_u8[reg_count]; + state->ext_isr.reg_u8[reg_count] = old_data_u8 & (~val_data_u8); + + mask = 0x1; + for (i = 0; i < 8; i++) { + if ((old_data_u8 & mask) && (val_data_u8 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 8, 0); + } + mask = mask << 1; + } + + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + /* percpu(32 bytes) coreisr reg_count is 0~31 */ + vcpu_id = (offset >> 8) & 0xff; + reg_count = offset & 0xff; + + state->ext_core_isr.reg_u8[vcpu_id][reg_count] &= ~val_data_u8; + + bits = sizeof(val_data_u8) * 8; + i = find_first_bit((void *)&val_data_u8, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u8, i, 1); + i = find_first_bit((void *)&val_data_u8, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + /*drop arch.core_ip_mask use state->ip_map*/ + reg_count = (offset - EXTIOI_IPMAP_START); + state->ip_map.reg_u8[reg_count] = val_data_u8; + + ipnum = 0; + + for (i = 0; i < 4; i++) { + if (val_data_u8 & (0x1 << i)) { + ipnum = i; + break; + } + } + + if (val_data_u8) { + for (i = 0; i < 32; i++) { + irqnum = reg_count * 32 + i; + state->ext_sw_ipmap[irqnum] = ipnum; + } + } else { + for (i = 0; i < 32; i++) { + irqnum = reg_count * 32 + i; + state->ext_sw_ipmap[irqnum] = 0; + } + } + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START); + state->core_map.reg_u8[reg_count] = val_data_u8; + state->ext_sw_coremap[reg_count] = val_data_u8; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START); + state->node_type.reg_u8[reg_count] = val_data_u8; + } else { + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx\n", + __FUNCTION__, addr); + } + + return 0; +} + +static int ls3a_ext_intctl_writew(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, reg_count; + uint32_t val_data_u32, old_data_u32, mask; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + uint8_t tmp_data_u8; + int i, level, vcpu_id; + unsigned long val; + + val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u32 = val & 0xffffffffUL; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 4; + old_data_u32 = state->ext_en.reg_u32[reg_count]; + if (old_data_u32 != val_data_u32) { + state->ext_en.reg_u32[reg_count] = val_data_u32; + old_data_u32 = old_data_u32 ^ val_data_u32; + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u32); i++) { + if (old_data_u32 & mask) { + level = !!(val_data_u32 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 32, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 4; + state->bounce.reg_u32[reg_count] = val_data_u32; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) / 4; + old_data_u32 = state->ext_isr.reg_u32[reg_count]; + state->ext_isr.reg_u32[reg_count] = old_data_u32 & (~val_data_u32); + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u32); i++) { + if ((old_data_u32 & mask) && (val_data_u32 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 32, 0); + } + mask = mask << 1; + } + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + /* percpu(32 bytes) coreisr reg_count is 0~7*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 4; + + /*ext_core_ioisr*/ + state->ext_core_isr.reg_u32[vcpu_id][reg_count] &= ~val_data_u32; + + bits = sizeof(val_data_u32) * 8; + i = find_first_bit((void *)&val_data_u32, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u32, i, 1); + i = find_first_bit((void *)&val_data_u32, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + tmp_data_u8 = val_data_u32 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + tmp_data_u8 = val_data_u32 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + kvm_debug("%s:id:%d addr=0x%llx, offset 0x%llx val 0x%x\n", + __FUNCTION__, vcpu->vcpu_id, addr, offset, val_data_u32); + + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 4; + state->node_type.reg_u32[reg_count] = val_data_u32; + } else { + WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n", + __FUNCTION__, __LINE__, addr); + } + + return 0; +} + +static int ls3a_ext_intctl_writel(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, val_data_u64, old_data_u64, reg_count, mask, i; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + uint8_t tmp_data_u8; + int level, vcpu_id; + + unsigned long val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u64 = val; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 8; + old_data_u64 = state->ext_en.reg_u64[reg_count]; + if (old_data_u64 != val_data_u64) { + state->ext_en.reg_u64[reg_count] = val_data_u64; + old_data_u64 = old_data_u64 ^ val_data_u64; + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u64); i++) { + if (old_data_u64 & mask) { + level = !!(val_data_u64 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 64, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 8; + state->bounce.reg_u64[reg_count] = val_data_u64; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) / 8; + old_data_u64 = state->ext_isr.reg_u64[reg_count]; + state->ext_isr.reg_u64[reg_count] = old_data_u64 & (~val_data_u64); + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u64); i++) { + if ((old_data_u64 & mask) && (val_data_u64 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 64, 0); + } + mask = mask << 1; + } + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0x1f) / 8; + + /*core_ext_ioisr*/ + state->ext_core_isr.reg_u64[vcpu_id][reg_count] &= ~val_data_u64; + + bits = sizeof(val_data_u64) * 8; + i = find_first_bit((void *)&val_data_u64, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u64, i, 1); + i = find_first_bit((void *)&val_data_u64, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + tmp_data_u8 = val_data_u64 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + + tmp_data_u8 = (val_data_u64 >> 32) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 40) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 48) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 56) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8); + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + tmp_data_u8 = val_data_u64 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + + tmp_data_u8 = (val_data_u64 >> 32) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 40) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 48) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 56) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8); + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 8; + state->node_type.reg_u64[reg_count] = val_data_u64; + } else { + WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n", + __FUNCTION__, __LINE__, addr); + } + return 0; +} +/** + * ls3a_ext_intctl_write() + * @kvm: KVM structure pointer + * @addr: Register address + * @size: The width of the register to be writen. + * @val: Value to be written. + * + * Analog extended interrupt related register write. + * + */ +static int ls3a_ext_intctl_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int size, const void *__val) +{ + struct ls3a_kvm_extirq *s = NULL; + unsigned long flags; + uint64_t offset; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + offset = addr & 0xfffff; + if (offset & (size - 1)) { + printk("%s(%d):unaligned address access %llx size %d\n", + __FUNCTION__, __LINE__, addr, size); + return 0; + } + + addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF; + ls3a_ext_irq_lock(s, flags); + + switch (size) { + case 1: + ls3a_ext_intctl_writeb(vcpu, dev, addr, __val); + break; + case 4: + ls3a_ext_intctl_writew(vcpu, dev, addr, __val); + break; + case 8: + ls3a_ext_intctl_writel(vcpu, dev, addr, __val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx,size %d\n", + __FUNCTION__, addr, size); + } + + ls3a_ext_irq_unlock(s, flags); + + kvm_debug("%s(%d):address access %llx size %d\n", + __FUNCTION__, __LINE__, offset, size); + return 0; +} + +static const struct kvm_io_device_ops kvm_ls3a_ext_irq_ops = { + .read = ls3a_ext_intctl_read, + .write = ls3a_ext_intctl_write, +}; + +void kvm_destroy_ls3a_ext_irq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *s = kvm->arch.v_extirq; + + if (!s) + return; + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(s->kvm, KVM_MMIO_BUS, &s->device); + mutex_unlock(&kvm->slots_lock); + kfree(s); +} +/* + * kvm_create_ls3a_ext_irq() + * @kvm KVM structure pointer + * Create an extended interrupt resource instance for a virtual machine + * Returns: Extended interrupt structure pointer + */ +int kvm_create_ls3a_ext_irq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *s; + int ret; + + s = kzalloc(sizeof(struct ls3a_kvm_extirq), GFP_KERNEL); + if (!s) + return -ENOMEM; + + memset((void *)&s->ls3a_ext_irq, 0x0, sizeof(struct kvm_ls3a_extirq_state)); + + spin_lock_init(&s->lock); + s->kvm = kvm; + + /* + * Initialize MMIO device + */ + kvm_iodevice_init(&s->device, &kvm_ls3a_ext_irq_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + EXTIOI_REG_BASE, EXTIOI_ADDR_SIZE, &s->device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + printk("%s dev_ls3a_ext_irq register error ret %d\n", __FUNCTION__, ret); + goto err_register; + } + + kvm->arch.v_extirq = s; + + return 0; + +err_register: + kfree(s); + return -EFAULT; +} + +static int kvm_set_ext_sw_ipmap(struct kvm_ls3a_extirq_state *state) +{ + uint8_t val_data_u8; + int i, j, base_irq, irqnum, ipnum; + + ipnum = 0; + for (i = 0; i < EXTIOI_IRQS_IPMAP_SIZE; i++) { + val_data_u8 = state->ip_map.reg_u8[i]; + for (j = 0; j < 4; j++) { + if (val_data_u8 & (0x1 << j)) { + ipnum = j; + break; + } + } + kvm_debug("%s:%d ipnum:%d i:%d val_data_u8:0x%x\n", __FUNCTION__, __LINE__, + ipnum, i, val_data_u8); + + if (val_data_u8) { + for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) { + irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq; + state->ext_sw_ipmap[irqnum] = ipnum; + } + } else { + for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) { + irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq; + state->ext_sw_ipmap[irqnum] = 0; + } + } + } + + return 0; +} + +static int kvm_set_ext_sw_coremap(struct kvm *kvm, struct kvm_ls3a_extirq_state *state) +{ + int reg_count; + + for (reg_count = 0; reg_count < EXTIOI_IRQS; reg_count++) { + state->ext_sw_coremap[reg_count] = state->core_map.reg_u8[reg_count]; + + kvm_debug("%s:%d -- reg_count:%d vcpu %d\n", + __FUNCTION__, __LINE__, reg_count, state->core_map.reg_u8[reg_count]); + } + + return 0; +} + +static int kvm_set_ext_sw_ipisr(struct kvm *kvm, struct kvm_ls3a_extirq_state *state) +{ + int ipnum, core, irq_num; + + for (irq_num = 0; irq_num < EXTIOI_IRQS; irq_num++) { + core = state->ext_sw_coremap[irq_num]; + ipnum = state->ext_sw_ipmap[irq_num]; + + if (test_bit(irq_num, (void *)state->ext_core_isr.reg_u8[core]) == false) { + bitmap_clear((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1); + } else { + bitmap_set((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1); + } + + } + return 0; +} + +int kvm_get_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memcpy(state, extirq_state, + sizeof(struct kvm_loongarch_ls3a_extirq_state)); + ls3a_ext_irq_unlock(v_extirq, flags); + kvm->stat.get_ls3a_ext_irq++; + + return 0; +} + +int kvm_set_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memcpy(extirq_state, state, + sizeof(struct kvm_loongarch_ls3a_extirq_state)); + kvm_set_ext_sw_ipmap(extirq_state); + kvm_set_ext_sw_coremap(kvm, extirq_state); + kvm_set_ext_sw_ipisr(kvm, extirq_state); + + ls3a_ext_irq_unlock(v_extirq, flags); + kvm->stat.set_ls3a_ext_irq++; + + return 0; +} + +int kvm_setup_ls3a_extirq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memset(extirq_state, 0, sizeof(struct kvm_ls3a_extirq_state)); + ls3a_ext_irq_unlock(v_extirq, flags); + + return 0; +} + +void kvm_dump_ls3a_extirq_state(struct seq_file *s, + struct ls3a_kvm_extirq *irqchip) +{ + struct kvm_ls3a_extirq_state *extirq; + int i, j = 0; + unsigned long flags; + + seq_puts(s, "LS3A ext irqchip state:\n"); + + if (!irqchip) + return; + + extirq = &(irqchip->ls3a_ext_irq); + ls3a_ext_irq_lock(irqchip, flags); + seq_puts(s, "ext irq enabled"); + seq_puts(s, "\nenabled:(Not Enabled)"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (!test_bit(i, (void *)&extirq->ext_en)) + seq_printf(s, "%d ", i); + } + seq_puts(s, "\nbounce:(Not bounce)"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (!test_bit(i, (void *)&extirq->bounce)) + seq_printf(s, "%d ", i); + } + seq_puts(s, "\next_isr:"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (test_bit(i, (void *)&extirq->ext_isr)) + seq_printf(s, "%d ", i); + } + + seq_puts(s, "\ncore_isr:"); + for (i = 0; i < KVM_MAX_VCPUS && kvm_get_vcpu_by_id(irqchip->kvm, i); i++) { + seq_printf(s, "\n\t CPU%d:", i); + for (j = 0; j < EXTIOI_IRQS; j++) { + if (test_bit(j, (void *)&extirq->ext_core_isr.reg_u8[i])) + seq_printf(s, "%d ", j); + } + } + seq_printf(s, "\nip_map:%llx", extirq->ip_map.reg_u64); + seq_puts(s, "\ncore_map: (only display router to slave cpu)\n"); + for (i = 0; i < EXTIOI_IRQS_COREMAP_SIZE; i++) + if (extirq->core_map.reg_u8[i]) + seq_printf(s, "\tirq:%d -> cpu:%d\n", i, + extirq->core_map.reg_u8[i]); + ls3a_ext_irq_unlock(irqchip, flags); +} diff --git a/arch/loongarch/kvm/intc/ls3a_ext_irq.h b/arch/loongarch/kvm/intc/ls3a_ext_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..075d78120b32cab4db73549292728d2da3f75d47 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ext_irq.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS3A_KVM_EXT_IRQ_H +#define __LS3A_KVM_EXT_IRQ_H + +#include +#include +#include +#include +#include + +#include + +#define IOCSR_EXTIOI_ADDR KVM_IOCSR_EXTIOI_NODEMAP_BASE + +#define EXTIOI_ADDR_OFF 0x10000 +#define EXTIOI_REG_BASE (LOONGSON_VIRT_REG_BASE + EXTIOI_ADDR_OFF) +#define EXTIOI_REG_END (EXTIOI_REG_BASE + 0x20000) +#define EXTIOI_ADDR_SIZE (EXTIOI_REG_END - EXTIOI_REG_BASE) +#define EXTIOI_PERCORE_REG_OFF 0x10000 +#define EXTIOI_PERCORE_REG_END (EXTIOI_PERCORE_REG_OFF + 0x10000) + +#define EXTIOI_ADDR(off) (EXTIOI_REG_BASE + (off) - IOCSR_EXTIOI_ADDR) +#define EXTIOI_PERCORE_ADDR(id, off) \ + (EXTIOI_REG_BASE + EXTIOI_PERCORE_REG_OFF + ((id) << 8) + (off)) + +#define EXTIOI_NODETYPE_START (KVM_IOCSR_EXTIOI_NODEMAP_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_NODETYPE_END (EXTIOI_NODETYPE_START + 0x20) +#define EXTIOI_IPMAP_START (KVM_IOCSR_EXTIOI_IPMAP_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_IPMAP_END (EXTIOI_IPMAP_START + 0x8) +#define EXTIOI_ENABLE_START (KVM_IOCSR_EXTIOI_EN_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_ENABLE_END (EXTIOI_ENABLE_START + 0x20) +#define EXTIOI_BOUNCE_START (KVM_IOCSR_EXTIOI_BOUNCE_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_BOUNCE_END (EXTIOI_BOUNCE_START + 0x20) +#define EXTIOI_ISR_START (0x1700 - IOCSR_EXTIOI_ADDR) +#define EXTIOI_ISR_END (EXTIOI_ISR_START + 0x20) +#define EXTIOI_COREMAP_START (KVM_IOCSR_EXTIOI_ROUTE_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_COREMAP_END (EXTIOI_COREMAP_START + 0x100) +#define EXTIOI_COREISR_START (EXTIOI_PERCORE_REG_OFF) +#define EXTIOI_COREISR_END (EXTIOI_PERCORE_REG_END) + +#define LS3A_INTC_IP 8 +#define EXTIOI_IRQS KVM_EXTIOI_IRQS +#define EXTIOI_IRQS_BITMAP_SIZE (EXTIOI_IRQS / 8) +/* map to ipnum per 32 irqs */ +#define EXTIOI_IRQS_IPMAP_SIZE (EXTIOI_IRQS / 32) +#define EXTIOI_IRQS_PER_GROUP KVM_EXTIOI_IRQS_PER_GROUP +#define EXTIOI_IRQS_COREMAP_SIZE (EXTIOI_IRQS) +#define EXTIOI_IRQS_NODETYPE_SIZE KVM_EXTIOI_IRQS_NODETYPE_SIZE + +typedef struct kvm_ls3a_extirq_state { + union ext_en { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } ext_en; + union bounce { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } bounce; + union ext_isr { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } ext_isr; + union ext_core_isr { + uint64_t reg_u64[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE]; + } ext_core_isr; + union ip_map { + uint64_t reg_u64; + uint32_t reg_u32[EXTIOI_IRQS_IPMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_IPMAP_SIZE]; + } ip_map; + union core_map { + uint64_t reg_u64[EXTIOI_IRQS_COREMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_COREMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_COREMAP_SIZE]; + } core_map; + union node_type { + uint64_t reg_u64[EXTIOI_IRQS_NODETYPE_SIZE / 4]; + uint32_t reg_u32[EXTIOI_IRQS_NODETYPE_SIZE / 2]; + uint16_t reg_u16[EXTIOI_IRQS_NODETYPE_SIZE]; + uint8_t reg_u8[EXTIOI_IRQS_NODETYPE_SIZE * 2]; + } node_type; + + /*software state */ + uint8_t ext_sw_ipmap[EXTIOI_IRQS]; + uint8_t ext_sw_coremap[EXTIOI_IRQS]; + uint8_t ext_sw_ipisr[KVM_MAX_VCPUS][LS3A_INTC_IP][EXTIOI_IRQS_BITMAP_SIZE]; +} LS3AExtirqState; + +struct ls3a_kvm_extirq { + spinlock_t lock; + struct kvm *kvm; + struct kvm_io_device device; + struct kvm_ls3a_extirq_state ls3a_ext_irq; +}; + +static inline struct ls3a_kvm_extirq *ls3a_ext_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_extirq; +} + +static inline int ls3a_extirq_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls3a_ext_irqchip(kvm) != NULL); + return ret; +} + + +void ext_irq_handler(struct kvm *kvm, int irq, int level); +int kvm_create_ls3a_ext_irq(struct kvm *kvm); +int kvm_get_ls3a_extirq(struct kvm *kvm, + struct kvm_loongarch_ls3a_extirq_state *state); +int kvm_set_ls3a_extirq(struct kvm *kvm, + struct kvm_loongarch_ls3a_extirq_state *state); +void kvm_destroy_ls3a_ext_irq(struct kvm *kvm); +void msi_irq_handler(struct kvm *kvm, int irq, int level); +int kvm_setup_ls3a_extirq(struct kvm *kvm); +void kvm_dump_ls3a_extirq_state(struct seq_file *m, struct ls3a_kvm_extirq *irqchip); +#endif diff --git a/arch/loongarch/kvm/intc/ls3a_ipi.c b/arch/loongarch/kvm/intc/ls3a_ipi.c new file mode 100644 index 0000000000000000000000000000000000000000..d5706b39172a10dcccf3f1e13190e0ae90d199fe --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ipi.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include "kvmcpu.h" +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +#define ls3a_gipi_lock(s, flags) spin_lock_irqsave(&s->lock, flags) +#define ls3a_gipi_unlock(s, flags) spin_unlock_irqrestore(&s->lock, flags) + +extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq); +int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action) +{ + struct kvm *kvm = vcpu->kvm; + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *s = &(ipi->ls3a_gipistate); + unsigned long flags; + struct kvm_loongarch_interrupt irq; + + kvm->stat.pip_write_exits++; + + ls3a_gipi_lock(ipi, flags); + if (s->core[cpu].status == 0) { + irq.cpu = cpu; + irq.irq = LARCH_INT_IPI; + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + } + + s->core[cpu].status |= action; + ls3a_gipi_unlock(ipi, flags); + + return 0; +} + +static int ls3a_gipi_writel(struct ls3a_kvm_ipi *ipi, gpa_t addr, + int len, const void *val) +{ + uint64_t data, offset; + struct kvm_loongarch_interrupt irq; + gipiState *s = &(ipi->ls3a_gipistate); + uint32_t cpu, action_data; + struct kvm *kvm; + void *pbuf; + int mailbox, action; + + kvm = ipi->kvm; + cpu = (addr >> 8) & 0xff; + + data = *(uint64_t *)val; + offset = addr & 0xFF; + + BUG_ON(offset & (len - 1)); + + switch (offset) { + case CORE0_STATUS_OFF: + printk("CORE0_SET_OFF Can't be write\n"); + + break; + case CORE0_EN_OFF: + s->core[cpu].en = data; + + break; + case CORE0_IPI_SEND: + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + action = (data & 0x1f); + action_data = (1 << action); + + if (s->core[cpu].status == 0) { + irq.cpu = cpu; + irq.irq = LARCH_INT_IPI; + + if (likely(kvm->vcpus[cpu])) { + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + } + } + s->core[cpu].status |= action_data; + break; + case CORE0_SET_OFF: + pr_info("CORE0_SET_OFF simulation is required\n"); + break; + case CORE0_CLEAR_OFF: + s->core[cpu].status &= ~data; + if (!s->core[cpu].status) { + irq.cpu = cpu; + irq.irq = -LARCH_INT_IPI; + if (likely(kvm->vcpus[cpu])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + else + kvm_err("Failed lower ipi irq target cpu:%d\n", cpu); + } + + break; + case CORE0_MAIL_SEND: + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + mailbox = ((data & 0xffffffff) >> 2) & 0x7; + pbuf = (void *)s->core[cpu].buf + mailbox * 4; + + *(unsigned int *)pbuf = (unsigned int)(data >> 32); + break; + case 0x20 ... 0x3c: + pbuf = (void *)s->core[cpu].buf + (offset - 0x20); + if (len == 1) + *(unsigned char *)pbuf = (unsigned char)data; + else if (len == 2) + *(unsigned short *)pbuf = (unsigned short)data; + else if (len == 4) + *(unsigned int *)pbuf = (unsigned int)data; + else if (len == 8) + *(unsigned long *)pbuf = (unsigned long)data; + + break; + default: + printk("ls3a_gipi_writel with unknown addr %llx \n", addr); + break; + } + return 0; +} + +static uint64_t ls3a_gipi_readl(struct ls3a_kvm_ipi *ipi, + gpa_t addr, int len, void *val) +{ + uint64_t offset; + uint64_t ret = 0; + + gipiState *s = &(ipi->ls3a_gipistate); + uint32_t cpu; + void *pbuf; + + cpu = (addr >> 8) & 0xff; + + offset = addr & 0xFF; + + BUG_ON(offset & (len - 1)); + switch (offset) { + case CORE0_STATUS_OFF: + ret = s->core[cpu].status; + break; + case CORE0_EN_OFF: + ret = s->core[cpu].en; + break; + case CORE0_SET_OFF: + ret = 0; + break; + case CORE0_CLEAR_OFF: + ret = 0; + break; + case 0x20 ... 0x3c: + pbuf = (void *)s->core[cpu].buf + (offset - 0x20); + if (len == 1) + ret = *(unsigned char *)pbuf; + else if (len == 2) + ret = *(unsigned short *)pbuf; + else if (len == 4) + ret = *(unsigned int *)pbuf; + else if (len == 8) + ret = *(unsigned long *)pbuf; + break; + default: + printk("ls3a_gipi_readl with unknown addr %llx \n", addr); + break; + } + + *(uint64_t *)val = ret; + + return ret; +} + +static int kvm_ls3a_ipi_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct ls3a_kvm_ipi *ipi; + ipi_io_device *ipi_device; + unsigned long flags; + + ipi_device = container_of(dev, ipi_io_device, device); + ipi = ipi_device->ipi; + ipi->kvm->stat.pip_write_exits++; + + ls3a_gipi_lock(ipi, flags); + ls3a_gipi_writel(ipi, addr, len, val); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + + +static int kvm_ls3a_ipi_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + struct ls3a_kvm_ipi *ipi; + ipi_io_device *ipi_device; + unsigned long flags; + + ipi_device = container_of(dev, ipi_io_device, device); + ipi = ipi_device->ipi; + ipi->kvm->stat.pip_read_exits++; + + ls3a_gipi_lock(ipi, flags); + ls3a_gipi_readl(ipi, addr, len, val); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + + +static const struct kvm_io_device_ops kvm_ls3a_ipi_ops = { + .read = kvm_ls3a_ipi_read, + .write = kvm_ls3a_ipi_write, +}; + +void kvm_destroy_ls3a_ipi(struct kvm *kvm) +{ + struct kvm_io_device *device; + struct ls3a_kvm_ipi *vipi = kvm->arch.v_gipi; + + if (!vipi) + return; + device = &vipi->dev_ls3a_ipi.device; + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(vipi->kvm, KVM_MMIO_BUS, device); + mutex_unlock(&kvm->slots_lock); + kfree(vipi); +} + +int kvm_create_ls3a_ipi(struct kvm *kvm) +{ + struct ls3a_kvm_ipi *s; + unsigned long addr; + struct kvm_io_device *device; + int ret; + + s = kzalloc(sizeof(struct ls3a_kvm_ipi), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + /* + * Initialize MMIO device + */ + device = &s->dev_ls3a_ipi.device; + kvm_iodevice_init(device, &kvm_ls3a_ipi_ops); + addr = SMP_MAILBOX; + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + addr, KVM_IOCSR_IPI_ADDR_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + kvm_err("%s Initialize MMIO dev err ret:%d\n", __func__, ret); + goto err; + } else { + s->dev_ls3a_ipi.ipi = s; + } + + kvm->arch.v_gipi = s; + return 0; + +err: + kfree(s); + return -EFAULT; +} + +int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state) +{ + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *ipi_state = &(ipi->ls3a_gipistate); + unsigned long flags; + + ls3a_gipi_lock(ipi, flags); + memcpy(state, ipi_state, sizeof(gipiState)); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + +int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state) +{ + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *ipi_state = &(ipi->ls3a_gipistate); + unsigned long flags; + + if (!ipi) + return -EINVAL; + + ls3a_gipi_lock(ipi, flags); + memcpy(ipi_state, state, sizeof(gipiState)); + ls3a_gipi_unlock(ipi, flags); + return 0; +} diff --git a/arch/loongarch/kvm/intc/ls3a_ipi.h b/arch/loongarch/kvm/intc/ls3a_ipi.h new file mode 100644 index 0000000000000000000000000000000000000000..ae65c7d1a458ee1c9a8abe099cc2c2a7ae5f6846 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ipi.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS3A_KVM_IPI_H +#define __LS3A_KVM_IPI_H + +#include +#include +#include +#include + +#include + +typedef struct gipi_single { + uint32_t status; + uint32_t en; + uint32_t set; + uint32_t clear; + uint64_t buf[4]; +} gipi_single; + +typedef struct gipiState { + gipi_single core[KVM_MAX_VCPUS]; +} gipiState; + +struct ls3a_kvm_ipi; + +typedef struct ipi_io_device { + struct ls3a_kvm_ipi *ipi; + struct kvm_io_device device; + int nodeNum; +} ipi_io_device; + +struct ls3a_kvm_ipi { + spinlock_t lock; + struct kvm *kvm; + gipiState ls3a_gipistate; + int nodeNum; + ipi_io_device dev_ls3a_ipi; +}; + +#define SMP_MAILBOX (LOONGSON_VIRT_REG_BASE + 0x0000) +#define KVM_IPI_REG_ADDRESS(id, off) (SMP_MAILBOX | (id << 8) | off) +#define KVM_IOCSR_IPI_ADDR_SIZE 0x10000 + +#define CORE0_STATUS_OFF 0x000 +#define CORE0_EN_OFF 0x004 +#define CORE0_SET_OFF 0x008 +#define CORE0_CLEAR_OFF 0x00c +#define CORE0_BUF_20 0x020 +#define CORE0_BUF_28 0x028 +#define CORE0_BUF_30 0x030 +#define CORE0_BUF_38 0x038 +#define CORE0_IPI_SEND 0x040 +#define CORE0_MAIL_SEND 0x048 + +static inline struct ls3a_kvm_ipi *ls3a_ipi_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_gipi; +} + +static inline int ls3a_ipi_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls3a_ipi_irqchip(kvm) != NULL); + return ret; +} + +int kvm_create_ls3a_ipi(struct kvm *kvm); +void kvm_destroy_ls3a_ipi(struct kvm *kvm); +int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state); +int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state); +int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action); +#endif diff --git a/arch/loongarch/kvm/intc/ls7a_irq.c b/arch/loongarch/kvm/intc/ls7a_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..8a044159124a43f1b16db9e1c391e60fc7d32d64 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls7a_irq.c @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags) +{ + unsigned long tmp; + spin_lock_irqsave(&s->lock, tmp); + *flags = tmp; +} + +void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags) +{ + unsigned long tmp; + tmp = *flags; + spin_unlock_irqrestore(&s->lock, tmp); +} + +static void kvm_ls7a_ioapic_raise(struct kvm *kvm, unsigned long mask) +{ + unsigned long irqnum, val; + struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *state; + struct kvm_loongarch_interrupt irq; + int i; + + state = &s->ls7a_ioapic; + irq.cpu = -1; + val = mask & state->intirr & (~state->int_mask); + val &= ~state->intisr; + for_each_set_bit(i, &val, 64) { + state->intisr |= 0x1ULL << i; + irqnum = state->htmsi_vector[i]; + kvm_debug("msi_irq_handler,%ld,up\n", irqnum); + msi_irq_handler(kvm, irqnum, 1); + } + + kvm->stat.ls7a_ioapic_update++; +} + +static void kvm_ls7a_ioapic_lower(struct kvm *kvm, unsigned long mask) +{ + unsigned long irqnum, val; + struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *state; + struct kvm_loongarch_interrupt irq; + int i; + + state = &s->ls7a_ioapic; + irq.cpu = -1; + val = mask & state->intisr; + for_each_set_bit(i, &val, 64) { + state->intisr &= ~(0x1ULL << i); + irqnum = state->htmsi_vector[i]; + kvm_debug("msi_irq_handler,%ld,down\n", irqnum); + msi_irq_handler(kvm, irqnum, 0); + } + + kvm->stat.ls7a_ioapic_update++; +} + +int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, bool line_status) +{ + if (!level) + return -1; + + kvm_debug("msi data is 0x%x\n", e->msi.data); + msi_irq_handler(kvm, e->msi.data, 1); + return 0; +} + +int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi) +{ + struct kvm_kernel_irq_routing_entry route; + + if (msi->flags != 0) + return -EINVAL; + + kvm->stat.ls7a_msi_irq++; + route.msi.address_lo = msi->address_lo; + route.msi.address_hi = msi->address_hi; + route.msi.data = msi->data; + + kvm_debug("msi data is 0x%x\n", route.msi.data); + return kvm_ls7a_set_msi(&route, kvm, + KVM_USERSPACE_IRQ_SOURCE_ID, 1, false); + +} + +int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level) +{ + struct ls7a_kvm_ioapic *s; + struct kvm_ls7a_ioapic_state *state; + uint64_t mask = 1ULL << irq; + s = ls7a_ioapic_irqchip(kvm); + state = &s->ls7a_ioapic; + BUG_ON(irq < 0 || irq >= LS7A_IOAPIC_NUM_PINS); + + if (state->intedge & mask) { + /* edge triggered */ + if (level) { + if ((state->last_intirr & mask) == 0) { + state->intirr |= mask; + kvm_ls7a_ioapic_raise(kvm, mask); + } + state->last_intirr |= mask; + } else + state->last_intirr &= ~mask; + } else { + /* level triggered */ + if (!!level) { + if ((state->intirr & mask) == 0) { + state->intirr |= mask; + kvm_ls7a_ioapic_raise(kvm, mask); + } + } else { + if (state->intirr & mask) { + state->intirr &= ~mask; + kvm_ls7a_ioapic_lower(kvm, mask); + } + } + } + kvm->stat.ls7a_ioapic_set_irq++; + return 0; +} + +static int ls7a_ioapic_reg_write(struct ls7a_kvm_ioapic *s, + gpa_t addr, int len, const void *val) +{ + struct kvm *kvm; + struct kvm_ls7a_ioapic_state *state; + int64_t offset_tmp; + uint64_t offset; + uint64_t old, himask, lowmask; + unsigned long data, flags; + + offset = addr & 0xfff; + kvm = s->kvm; + state = &(s->ls7a_ioapic); + lowmask = 0xFFFFFFFFUL; + himask = lowmask << 32; + + if (offset & (len - 1)) { + printk("%s(%d):unaligned address access %llx size %d \n", + __FUNCTION__, __LINE__, addr, len); + return 0; + } + + if (8 == len) { + data = *(uint64_t *)val; + switch (offset) { + case LS7A_INT_MASK_OFFSET: + old = state->int_mask; + state->int_mask = data; + if (old & ~data) + kvm_ls7a_ioapic_raise(kvm, old & ~data); + if (~old & data) + kvm_ls7a_ioapic_lower(kvm, ~old & data); + break; + case LS7A_INT_STATUS_OFFSET: + state->intisr = data; + break; + case LS7A_INT_EDGE_OFFSET: + state->intedge = data; + break; + case LS7A_INT_CLEAR_OFFSET: + /* + * For emulated device, only clear edge triggered irq + * on writing INTCLR reg no effect on level triggered + * irq + * However for pass-through device with level-triggered + * intx, here need clear interrupt + */ + old = data; + data = data & state->intedge; + state->intirr &= ~data; + kvm_ls7a_ioapic_lower(kvm, data); + state->intisr &= ~data; + + data = old & ~state->intedge; + ls7a_ioapic_unlock(kvm->arch.v_ioapic, &flags); + for_each_set_bit(offset_tmp, &data, 64) + kvm_notify_acked_irq(kvm, 0, offset_tmp); + ls7a_ioapic_lock(kvm->arch.v_ioapic, &flags); + break; + case LS7A_INT_POL_OFFSET: + state->int_polarity = data; + break; + case LS7A_HTMSI_EN_OFFSET: + state->htmsi_en = data; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET: + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + } else if (4 == len) { + data = *(uint32_t *)val; + switch (offset) { + case LS7A_INT_MASK_OFFSET: + old = state->int_mask & lowmask; + state->int_mask = (state->int_mask & himask) | data; + if (old & ~data) + kvm_ls7a_ioapic_raise(kvm, old & ~data); + if (~old & data) + kvm_ls7a_ioapic_lower(kvm, ~old & data); + break; + case LS7A_INT_MASK_OFFSET + 4: + data = data << 32; + old = state->int_mask & himask; + state->int_mask = (state->int_mask & lowmask) | data; + if (old & ~data) + kvm_ls7a_ioapic_raise(kvm, old & ~data); + if (~old & data) + kvm_ls7a_ioapic_lower(kvm, ~old & data); + break; + case LS7A_INT_STATUS_OFFSET: + state->intisr = (state->intisr & himask) | data; + break; + case LS7A_INT_STATUS_OFFSET + 4: + data = data << 32; + state->intisr = (state->intisr & lowmask) | data; + break; + case LS7A_INT_EDGE_OFFSET: + state->intedge = (state->intedge & himask) | data; + break; + case LS7A_INT_EDGE_OFFSET + 4: + data = data << 32; + state->intedge = (state->intedge & lowmask) | data; + break; + case LS7A_INT_CLEAR_OFFSET + 4: + data = data << 32; + fallthrough; + case LS7A_INT_CLEAR_OFFSET: + old = data; + data = data & state->intedge; + state->intirr &= ~data; + kvm_ls7a_ioapic_lower(kvm, data); + state->intisr &= ~data; + + data = old & ~state->intedge; + ls7a_ioapic_unlock(kvm->arch.v_ioapic, &flags); + for_each_set_bit(offset_tmp, &data, 64) + kvm_notify_acked_irq(kvm, 0, offset_tmp); + ls7a_ioapic_lock(kvm->arch.v_ioapic, &flags); + break; + case LS7A_INT_POL_OFFSET: + state->int_polarity = (state->int_polarity & himask) | data; + break; + case LS7A_INT_POL_OFFSET+4: + data = data << 32; + state->int_polarity = (state->int_polarity & lowmask) | data; + break; + case LS7A_HTMSI_EN_OFFSET: + state->htmsi_en = (state->htmsi_en & himask) | data; + break; + case LS7A_HTMSI_EN_OFFSET+4: + data = data << 32; + state->htmsi_en = (state->htmsi_en & lowmask) | data; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL0_OFFSET+4: + case LS7A_AUTO_CTRL1_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET+4: + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + } else if (1 == len) { + data = *(unsigned char *)val; + if (offset >= LS7A_HTMSI_VEC_OFFSET) { + offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + state->htmsi_vector[offset_tmp] = + (uint8_t)(data & 0xff); + } + } else if (offset >= LS7A_ROUTE_ENTRY_OFFSET) { + offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + state->route_entry[offset_tmp] = + (uint8_t)(data & 0xff); + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + kvm->stat.ioapic_reg_write++; + return 0; +} + +static inline struct ls7a_kvm_ioapic *to_ioapic(struct kvm_io_device *dev) +{ + return container_of(dev, struct ls7a_kvm_ioapic, dev_ls7a_ioapic); +} + +static int kvm_ls7a_ioapic_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, + gpa_t addr, int len, const void *val) +{ + struct ls7a_kvm_ioapic *s = to_ioapic(this); + unsigned long flags; + + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_write(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + + return 0; +} + +static int ls7a_ioapic_reg_read(struct ls7a_kvm_ioapic *s, + gpa_t addr, int len, void *val) +{ + uint64_t offset, offset_tmp; + struct kvm *kvm; + struct kvm_ls7a_ioapic_state *state; + uint64_t result = 0, lowmask, himask; + + state = &(s->ls7a_ioapic); + kvm = s->kvm; + offset = addr & 0xfff; + lowmask = 0xFFFFFFFFUL; + himask = lowmask << 32; + if (offset & (len - 1)) { + printk("%s(%d):unaligned address access %llx size %d \n", + __FUNCTION__, __LINE__, addr, len); + return 0; + } + + if (8 == len) { + switch (offset) { + case LS7A_INT_MASK_OFFSET: + result = state->int_mask; + break; + case LS7A_INT_STATUS_OFFSET: + result = state->intisr & (~state->int_mask); + break; + case LS7A_INT_EDGE_OFFSET: + result = state->intedge; + break; + case LS7A_INT_POL_OFFSET: + result = state->int_polarity; + break; + case LS7A_HTMSI_EN_OFFSET: + result = state->htmsi_en; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET: + break; + case LS7A_INT_ID_OFFSET: + result = LS7A_INT_ID_VER; + result = (result << 32) + LS7A_INT_ID_VAL; + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + if (val != NULL) + *(uint64_t *)val = result; + } else if (4 == len) { + switch (offset) { + case LS7A_INT_MASK_OFFSET: + result = state->int_mask & lowmask; + break; + case LS7A_INT_MASK_OFFSET + 4: + result = state->int_mask & himask; + result = result >> 32; + break; + case LS7A_INT_STATUS_OFFSET: + result = state->intisr & (~state->int_mask) & lowmask; + break; + case LS7A_INT_STATUS_OFFSET + 4: + result = state->intisr & (~state->int_mask) & himask; + result = result >> 32; + break; + case LS7A_INT_EDGE_OFFSET: + result = state->intedge & lowmask; + break; + case LS7A_INT_EDGE_OFFSET + 4: + result = state->intedge & himask; + result = result >> 32; + break; + case LS7A_INT_POL_OFFSET: + result = state->int_polarity & lowmask; + break; + case LS7A_INT_POL_OFFSET + 4: + result = state->int_polarity & himask; + result = result >> 32; + break; + case LS7A_HTMSI_EN_OFFSET: + result = state->htmsi_en & lowmask; + break; + case LS7A_HTMSI_EN_OFFSET + 4: + result = state->htmsi_en & himask; + result = result >> 32; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL0_OFFSET + 4: + case LS7A_AUTO_CTRL1_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET + 4: + break; + case LS7A_INT_ID_OFFSET: + result = LS7A_INT_ID_VAL; + break; + case LS7A_INT_ID_OFFSET + 4: + result = LS7A_INT_ID_VER; + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + if (val != NULL) + *(uint32_t *)val = result; + } else if (1 == len) { + if (offset >= LS7A_HTMSI_VEC_OFFSET) { + offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + result = state->htmsi_vector[offset_tmp]; + } + } else if (offset >= LS7A_ROUTE_ENTRY_OFFSET) { + offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + result = state->route_entry[offset_tmp]; + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + if (val != NULL) + *(unsigned char *)val = result; + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + kvm->stat.ioapic_reg_read++; + return result; +} + +static int kvm_ls7a_ioapic_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, + gpa_t addr, int len, void *val) +{ + struct ls7a_kvm_ioapic *s = to_ioapic(this); + unsigned long flags; + uint64_t result = 0; + + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + result = ls7a_ioapic_reg_read(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + return 0; +} + +static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops = { + .read = kvm_ls7a_ioapic_read, + .write = kvm_ls7a_ioapic_write, +}; + +static int kvm_ls7a_ioapic_alias_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, gpa_t addr, int len, void *val) +{ + struct ls7a_kvm_ioapic *s; + unsigned long flags; + + s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias); + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_read(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + return 0; +} + +static int kvm_ls7a_ioapic_alias_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, gpa_t addr, int len, const void *val) +{ + struct ls7a_kvm_ioapic *s; + unsigned long flags; + + s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias); + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_write(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + + return 0; +} + +static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops_alias = { + .read = kvm_ls7a_ioapic_alias_read, + .write = kvm_ls7a_ioapic_alias_write, +}; + +int kvm_create_ls7a_ioapic(struct kvm *kvm) +{ + struct ls7a_kvm_ioapic *s; + int ret; + unsigned long ls7a_ioapic_reg_base; + + s = kzalloc(sizeof(struct ls7a_kvm_ioapic), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE; + + /* + * Initialize MMIO device + */ + kvm_iodevice_init(&s->dev_ls7a_ioapic, &kvm_ls7a_ioapic_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base, + 0x1000, &s->dev_ls7a_ioapic); + if (ret < 0) { + kvm_err("Failed register ioapic, err:%d\n", ret); + goto fail_unlock; + } + + ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE_ALIAS; + kvm_iodevice_init(&s->ls7a_ioapic_alias, &kvm_ls7a_ioapic_ops_alias); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base, + 0x1000, &s->ls7a_ioapic_alias); + if (ret < 0) { + kvm_err("Failed register alias ioapic, err:%d\n", ret); + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, + &s->dev_ls7a_ioapic); + goto fail_unlock; + } + mutex_unlock(&kvm->slots_lock); + + kvm->arch.v_ioapic = s; + + return 0; + +fail_unlock: + mutex_unlock(&kvm->slots_lock); + kfree(s); + + return -EFAULT; +} + + +int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state) +{ + struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *ioapic_state = + &(ls7a_ioapic->ls7a_ioapic); + unsigned long flags; + + ls7a_ioapic_lock(ls7a_ioapic, &flags); + memcpy(state, ioapic_state, sizeof(struct kvm_ls7a_ioapic_state)); + ls7a_ioapic_unlock(ls7a_ioapic, &flags); + kvm->stat.get_ls7a_ioapic++; + return 0; +} + +int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state) +{ + struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *ioapic_state = + &(ls7a_ioapic->ls7a_ioapic); + unsigned long flags; + + if (!ls7a_ioapic) + return -EINVAL; + + ls7a_ioapic_lock(ls7a_ioapic, &flags); + memcpy(ioapic_state, state, sizeof(struct kvm_ls7a_ioapic_state)); + ls7a_ioapic_unlock(ls7a_ioapic, &flags); + kvm->stat.set_ls7a_ioapic++; + return 0; +} + +void kvm_destroy_ls7a_ioapic(struct kvm *kvm) +{ + struct ls7a_kvm_ioapic *vpic = kvm->arch.v_ioapic; + if (!vpic) + return; + mutex_lock(&kvm->slots_lock); + kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS, + &vpic->ls7a_ioapic_alias); + kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS, + &vpic->dev_ls7a_ioapic); + mutex_unlock(&kvm->slots_lock); + kfree(vpic); +} + +void kvm_dump_ls7a_ioapic_state(struct seq_file *m, + struct ls7a_kvm_ioapic *ioapic) +{ + struct kvm_ls7a_ioapic_state *ioapic_state; + unsigned long flags; + int i = 0; + + if (!ioapic) + return; + + seq_puts(m, "\nIOAPIC state:\n"); + ioapic_state = &(ioapic->ls7a_ioapic); + + ls7a_ioapic_lock(ioapic, &flags); + seq_puts(m, "irq masked: "); + for (i = 0; i < 64; i++) { + if (!test_bit(i, (void *)&ioapic_state->int_mask)) + seq_printf(m, "%d ", i); + } + seq_printf(m, "\nhtmsi_en:0x%016llx\n" + "intedge:0x%016llx", + ioapic_state->htmsi_en, + ioapic_state->intedge); + + seq_puts(m, "\nroute_entry: "); + for (i = 0; i < 64; i++) + seq_printf(m, "%d ", ioapic_state->route_entry[i]); + + seq_puts(m, "\nhtmsi_vector: "); + for (i = 0; i < 64; i++) + seq_printf(m, "%d ", ioapic_state->htmsi_vector[i]); + + seq_printf(m, "\nintirr:%016llx\n" + "intisr:%016llx\n", + ioapic_state->intirr, + ioapic_state->intisr); + ls7a_ioapic_unlock(ioapic, &flags); +} diff --git a/arch/loongarch/kvm/intc/ls7a_irq.h b/arch/loongarch/kvm/intc/ls7a_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..1bb7d3d8a74d3c4e09f95dc61452a4b07a2adba4 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls7a_irq.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS7A_KVM_IRQ_H +#define __LS7A_KVM_IRQ_H + +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" + +#include + +#define LS7A_APIC_NUM_PINS 64 + +#define LS7A_ROUTE_ENTRY_OFFSET 0x100 +#define LS7A_INT_ID_OFFSET 0x0 +#define LS7A_INT_ID_VAL 0x7000000UL +#define LS7A_INT_ID_VER 0x1f0001UL +#define LS7A_INT_MASK_OFFSET 0x20 +#define LS7A_INT_EDGE_OFFSET 0x60 +#define LS7A_INT_CLEAR_OFFSET 0x80 +#define LS7A_INT_STATUS_OFFSET 0x3a0 +#define LS7A_INT_POL_OFFSET 0x3e0 +#define LS7A_HTMSI_EN_OFFSET 0x40 +#define LS7A_HTMSI_VEC_OFFSET 0x200 +#define LS7A_AUTO_CTRL0_OFFSET 0xc0 +#define LS7A_AUTO_CTRL1_OFFSET 0xe0 + +#define LS7A_IOAPIC_GUEST_REG_BASE 0x10000000UL +#define LS7A_IOAPIC_GUEST_REG_BASE_ALIAS 0xe0010000000UL + +#define LS7A_IOAPIC_NUM_PINS 32 + +typedef struct kvm_ls7a_ioapic_state { + u64 int_id; + /* 0x020 interrupt mask register */ + u64 int_mask; + /* 0x040 1=msi */ + u64 htmsi_en; + /* 0x060 edge=1 level =0 */ + u64 intedge; + /* 0x080 for clean edge int,set 1 clean,set 0 is noused */ + u64 intclr; + /* 0x0c0 */ + u64 auto_crtl0; + /* 0x0e0 */ + u64 auto_crtl1; + /* 0x100 - 0x140 */ + u8 route_entry[64]; + /* 0x200 - 0x240 */ + u8 htmsi_vector[64]; + /* 0x300 */ + u64 intisr_chip0; + /* 0x320 */ + u64 intisr_chip1; + /* edge detection */ + u64 last_intirr; + /* 0x380 interrupt request register */ + u64 intirr; + /* 0x3a0 interrupt service register */ + u64 intisr; + /* 0x3e0 interrupt level polarity selection register, + * 0 for high level tirgger + */ + u64 int_polarity; +} LS7AApicState; + +struct ls7a_kvm_ioapic { + spinlock_t lock; + bool wakeup_needed; + unsigned pending_acks; + struct kvm *kvm; + struct kvm_ls7a_ioapic_state ls7a_ioapic; + struct kvm_io_device dev_ls7a_ioapic; + struct kvm_io_device ls7a_ioapic_alias; + void (*ack_notifier)(void *opaque, int irq); + unsigned long irq_states[LS7A_APIC_NUM_PINS]; +}; + +static inline struct ls7a_kvm_ioapic *ls7a_ioapic_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_ioapic; +} + +static inline int ls7a_ioapic_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls7a_ioapic_irqchip(kvm) != NULL); + return ret; +} + +int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state); +int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state); +int kvm_create_ls7a_ioapic(struct kvm *kvm); +void kvm_destroy_ls7a_ioapic(struct kvm *kvm); +int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level); + +void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags); +void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags); +int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); +int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status); + +void kvm_dump_ls7a_ioapic_state(struct seq_file *m, struct ls7a_kvm_ioapic *ioapic); +#endif diff --git a/arch/loongarch/kvm/interrupt.c b/arch/loongarch/kvm/interrupt.c new file mode 100644 index 0000000000000000000000000000000000000000..fb7c4ef7f21f31310546255899c06c48d1ffee54 --- /dev/null +++ b/arch/loongarch/kvm/interrupt.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include +#include "kvm_compat.h" + +static u32 int_to_coreint[LOONGARCH_EXC_MAX] = { + [LARCH_INT_TIMER] = CPU_TIMER, + [LARCH_INT_IPI] = CPU_IPI, + [LARCH_INT_SIP0] = CPU_SIP0, + [LARCH_INT_SIP1] = CPU_SIP1, + [LARCH_INT_IP0] = CPU_IP0, + [LARCH_INT_IP1] = CPU_IP1, + [LARCH_INT_IP2] = CPU_IP2, + [LARCH_INT_IP3] = CPU_IP3, + [LARCH_INT_IP4] = CPU_IP4, + [LARCH_INT_IP5] = CPU_IP5, + [LARCH_INT_IP6] = CPU_IP6, + [LARCH_INT_IP7] = CPU_IP7, +}; + +static int _kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority) +{ + unsigned int irq = 0; + + clear_bit(priority, &vcpu->arch.irq_pending); + if (priority < LOONGARCH_EXC_MAX) + irq = int_to_coreint[priority]; + + switch (priority) { + case LARCH_INT_TIMER: + case LARCH_INT_IPI: + case LARCH_INT_SIP0: + case LARCH_INT_SIP1: + kvm_set_gcsr_estat(irq); + break; + + case LARCH_INT_IP0: + case LARCH_INT_IP1: + case LARCH_INT_IP2: + case LARCH_INT_IP3: + case LARCH_INT_IP4: + case LARCH_INT_IP5: + case LARCH_INT_IP6: + case LARCH_INT_IP7: + kvm_set_csr_gintc(irq); + break; + + default: + break; + } + + return 1; +} + +static int _kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority) +{ + unsigned int irq = 0; + + clear_bit(priority, &vcpu->arch.irq_clear); + if (priority < LOONGARCH_EXC_MAX) + irq = int_to_coreint[priority]; + + switch (priority) { + case LARCH_INT_TIMER: + case LARCH_INT_IPI: + case LARCH_INT_SIP0: + case LARCH_INT_SIP1: + kvm_clear_gcsr_estat(irq); + break; + + case LARCH_INT_IP0: + case LARCH_INT_IP1: + case LARCH_INT_IP2: + case LARCH_INT_IP3: + case LARCH_INT_IP4: + case LARCH_INT_IP5: + case LARCH_INT_IP6: + case LARCH_INT_IP7: + kvm_clear_csr_gintc(irq); + break; + + default: + break; + } + + return 1; +} + +void _kvm_deliver_intr(struct kvm_vcpu *vcpu) +{ + unsigned long *pending = &vcpu->arch.irq_pending; + unsigned long *pending_clr = &vcpu->arch.irq_clear; + unsigned int priority; + + if (!(*pending) && !(*pending_clr)) + return; + + if (*pending_clr) { + priority = __ffs(*pending_clr); + while (priority <= LOONGARCH_EXC_IPNUM) { + _kvm_irq_clear(vcpu, priority); + priority = find_next_bit(pending_clr, + BITS_PER_BYTE * sizeof(*pending_clr), + priority + 1); + } + } + + if (*pending) { + priority = __ffs(*pending); + while (priority <= LOONGARCH_EXC_IPNUM) { + _kvm_irq_deliver(vcpu, priority); + priority = find_next_bit(pending, + BITS_PER_BYTE * sizeof(*pending), + priority + 1); + } + } + +} + +int _kvm_pending_timer(struct kvm_vcpu *vcpu) +{ + return test_bit(LARCH_INT_TIMER, &vcpu->arch.irq_pending); +} diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c new file mode 100644 index 0000000000000000000000000000000000000000..fef3af5309694750b7ab2c9332ad4abd2c5d4f40 --- /dev/null +++ b/arch/loongarch/kvm/irqfd.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include "intc/ls7a_irq.h" + +static int kvm_ls7a_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + /* ioapic pin (0 ~ 64) <---> gsi(0 ~ 64) */ + u32 irq = e->irqchip.pin; + unsigned long flags; + int ret; + + if (!ls7a_ioapic_in_kernel(kvm)) + return -ENXIO; + + ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags); + ret = kvm_ls7a_ioapic_set_irq(kvm, irq, level); + ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags); + + return ret; +} + +/** + * kvm_set_routing_entry: populate a kvm routing entry + * from a user routing entry + * + * @kvm: the VM this entry is applied to + * @e: kvm kernel routing entry handle + * @ue: user api routing entry handle + * return 0 on success, -EINVAL on errors. + */ +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + int r = -EINVAL; + + switch (ue->type) { + case KVM_IRQ_ROUTING_IRQCHIP: + e->set = kvm_ls7a_set_ioapic_irq; + + e->irqchip.irqchip = ue->u.irqchip.irqchip; + e->irqchip.pin = ue->u.irqchip.pin; + + if (e->irqchip.pin >= LS7A_APIC_NUM_PINS) + goto out; + break; + case KVM_IRQ_ROUTING_MSI: + e->set = kvm_set_msi; + e->msi.address_lo = ue->u.msi.address_lo; + e->msi.address_hi = ue->u.msi.address_hi; + e->msi.data = ue->u.msi.data; + /* + * linux4.19 kernel have flags and devid + * e->msi.flags = ue->flags; + * e->msi.devid = ue->u.msi.devid; + */ + break; + default: + goto out; + } + r = 0; +out: + return r; +} + +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + if (e->type == KVM_IRQ_ROUTING_MSI) + return kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false); + + return -EWOULDBLOCK; +} + +/** + * kvm_set_msi: inject the MSI corresponding to the + * MSI routing entry + * + * This is the entry point for irqfd MSI injection + * and userspace MSI injection. + */ +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + unsigned int ret; + + if (!level) + return -1; + + ret = kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false); + return ret; +} + +int kvm_ls7a_setup_default_irq_routing(struct kvm *kvm) +{ + struct kvm_irq_routing_entry *entries; + + u32 nr = LS7A_APIC_NUM_PINS; + int i, ret; + + entries = kcalloc(nr, sizeof(*entries), GFP_KERNEL); + if (!entries) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + entries[i].gsi = i; + entries[i].type = KVM_IRQ_ROUTING_IRQCHIP; + entries[i].u.irqchip.irqchip = 0; + entries[i].u.irqchip.pin = i; + } + ret = kvm_set_irq_routing(kvm, entries, nr, 0); + kfree(entries); + + return 0; +} diff --git a/arch/loongarch/kvm/kvm_compat.h b/arch/loongarch/kvm/kvm_compat.h new file mode 100644 index 0000000000000000000000000000000000000000..b497a1b748603b397c8515f424ca4fb7b3c921e8 --- /dev/null +++ b/arch/loongarch/kvm/kvm_compat.h @@ -0,0 +1,762 @@ +#ifndef __LOONGARCH_KVM_COMPAT_H__ +#define __LOONGARCH_KVM_COMPAT_H__ + +#ifdef __ASSEMBLY__ +#define _ULCAST_ +#else +#define _ULCAST_ (unsigned long) +#endif + +#define KVM_CSR_CRMD 0x0 /* Current mode info */ +#define KVM_CRMD_WE_SHIFT 9 +#define KVM_CRMD_WE (_ULCAST_(0x1) << KVM_CRMD_WE_SHIFT) +#define KVM_CRMD_DACM_SHIFT 7 +#define KVM_CRMD_DACM_WIDTH 2 +#define KVM_CRMD_DACM (_ULCAST_(0x3) << KVM_CRMD_DACM_SHIFT) +#define KVM_CRMD_DACF_SHIFT 5 +#define KVM_CRMD_DACF_WIDTH 2 +#define KVM_CRMD_DACF (_ULCAST_(0x3) << KVM_CRMD_DACF_SHIFT) +#define KVM_CRMD_PG_SHIFT 4 +#define KVM_CRMD_PG (_ULCAST_(0x1) << KVM_CRMD_PG_SHIFT) +#define KVM_CRMD_DA_SHIFT 3 +#define KVM_CRMD_DA (_ULCAST_(0x1) << KVM_CRMD_DA_SHIFT) +#define KVM_CRMD_IE_SHIFT 2 +#define KVM_CRMD_IE (_ULCAST_(0x1) << KVM_CRMD_IE_SHIFT) +#define KVM_CRMD_PLV_SHIFT 0 +#define KVM_CRMD_PLV_WIDTH 2 +#define KVM_CRMD_PLV (_ULCAST_(0x3) << KVM_CRMD_PLV_SHIFT) + +#define KVM_CSR_PRMD 0x1 /* Prev-exception mode info */ +#define KVM_PRMD_PIE_SHIFT 2 +#define KVM_PRMD_PWE_SHIFT 3 +#define KVM_PRMD_PIE (_ULCAST_(0x1) << KVM_PRMD_PIE_SHIFT) +#define KVM_PRMD_PWE (_ULCAST_(0x1) << KVM_PRMD_PWE_SHIFT) +#define KVM_PRMD_PPLV_SHIFT 0 +#define KVM_PRMD_PPLV_WIDTH 2 +#define KVM_PRMD_PPLV (_ULCAST_(0x3) << KVM_PRMD_PPLV_SHIFT) + +#define KVM_CSR_EUEN 0x2 /* Extended unit enable */ +#define KVM_EUEN_LBTEN_SHIFT 3 +#define KVM_EUEN_LBTEN (_ULCAST_(0x1) << KVM_EUEN_LBTEN_SHIFT) +#define KVM_EUEN_LASXEN_SHIFT 2 +#define KVM_EUEN_LASXEN (_ULCAST_(0x1) << KVM_EUEN_LASXEN_SHIFT) +#define KVM_EUEN_LSXEN_SHIFT 1 +#define KVM_EUEN_LSXEN (_ULCAST_(0x1) << KVM_EUEN_LSXEN_SHIFT) +#define KVM_EUEN_FPEN_SHIFT 0 +#define KVM_EUEN_FPEN (_ULCAST_(0x1) << KVM_EUEN_FPEN_SHIFT) + +#define KVM_CSR_MISC 0x3 /* Misc config */ +#define KVM_CSR_ECFG 0x4 /* Exception config */ +#define KVM_ECFG_VS_SHIFT 16 +#define KVM_ECFG_VS_WIDTH 3 +#define KVM_ECFG_VS (_ULCAST_(0x7) << KVM_ECFG_VS_SHIFT) +#define KVM_ECFG_IM_SHIFT 0 +#define KVM_ECFG_IM_WIDTH 13 +#define KVM_ECFG_IM (_ULCAST_(0x1fff) << KVM_ECFG_IM_SHIFT) + +#define KVM_CSR_ESTAT 0x5 /* Exception status */ +#define KVM_ESTAT_ESUBCODE_SHIFT 22 +#define KVM_ESTAT_ESUBCODE_WIDTH 9 +#define KVM_ESTAT_ESUBCODE (_ULCAST_(0x1ff) << KVM_ESTAT_ESUBCODE_SHIFT) +#define KVM_ESTAT_EXC_SHIFT 16 +#define KVM_ESTAT_EXC_WIDTH 6 +#define KVM_ESTAT_EXC (_ULCAST_(0x3f) << KVM_ESTAT_EXC_SHIFT) +#define KVM_ESTAT_IS_SHIFT 0 +#define KVM_ESTAT_IS_WIDTH 15 +#define KVM_ESTAT_IS (_ULCAST_(0x7fff) << KVM_ESTAT_IS_SHIFT) + +#define KVM_CSR_ERA 0x6 /* ERA */ +#define KVM_CSR_BADV 0x7 /* Bad virtual address */ +#define KVM_CSR_BADI 0x8 /* Bad instruction */ +#define KVM_CSR_EENTRY 0xc /* Exception entry base address */ +#define KVM_CSR_TLBIDX 0x10 /* TLB Index, EHINV, PageSize, NP */ +#define KVM_CSR_TLBEHI 0x11 /* TLB EntryHi */ +#define KVM_CSR_TLBELO0 0x12 /* TLB EntryLo0 */ +#define KVM_CSR_TLBELO1 0x13 /* TLB EntryLo1 */ +#define KVM_CSR_GTLBC 0x15 /* Guest TLB control */ +#define KVM_GTLBC_TGID_SHIFT 16 +#define KVM_GTLBC_TGID_WIDTH 8 +#define KVM_GTLBC_TGID (_ULCAST_(0xff) << KVM_GTLBC_TGID_SHIFT) +#define KVM_GTLBC_TOTI_SHIFT 13 +#define KVM_GTLBC_TOTI (_ULCAST_(0x1) << KVM_GTLBC_TOTI_SHIFT) +#define KVM_GTLBC_USETGID_SHIFT 12 +#define KVM_GTLBC_USETGID (_ULCAST_(0x1) << KVM_GTLBC_USETGID_SHIFT) +#define KVM_GTLBC_GMTLBSZ_SHIFT 0 +#define KVM_GTLBC_GMTLBSZ_WIDTH 6 +#define KVM_GTLBC_GMTLBSZ (_ULCAST_(0x3f) << KVM_GTLBC_GMTLBSZ_SHIFT) + +#define KVM_CSR_TRGP 0x16 /* TLBR read guest info */ +#define KVM_CSR_ASID 0x18 /* ASID */ +#define KVM_CSR_PGDL 0x19 /* Page table base address when VA[47] = 0 */ +#define KVM_CSR_PGDH 0x1a /* Page table base address when VA[47] = 1 */ +#define KVM_CSR_PGD 0x1b /* Page table base */ +#define KVM_CSR_PWCTL0 0x1c /* PWCtl0 */ +#define KVM_CSR_PWCTL1 0x1d /* PWCtl1 */ +#define KVM_CSR_STLBPGSIZE 0x1e +#define KVM_CSR_RVACFG 0x1f +#define KVM_CSR_CPUID 0x20 /* CPU core number */ +#define KVM_CSR_PRCFG1 0x21 /* Config1 */ +#define KVM_CSR_PRCFG2 0x22 /* Config2 */ +#define KVM_CSR_PRCFG3 0x23 /* Config3 */ +#define KVM_CSR_KS0 0x30 +#define KVM_CSR_KS1 0x31 +#define KVM_CSR_KS2 0x32 +#define KVM_CSR_KS3 0x33 +#define KVM_CSR_KS4 0x34 +#define KVM_CSR_KS5 0x35 +#define KVM_CSR_KS6 0x36 +#define KVM_CSR_KS7 0x37 +#define KVM_CSR_KS8 0x38 +#define KVM_CSR_TMID 0x40 /* Timer ID */ +#define KVM_CSR_TCFG 0x41 /* Timer config */ +#define KVM_TCFG_VAL_SHIFT 2 +#define KVM_TCFG_VAL_WIDTH 48 +#define KVM_TCFG_VAL (_ULCAST_(0x3fffffffffff) << KVM_TCFG_VAL_SHIFT) +#define KVM_TCFG_PERIOD_SHIFT 1 +#define KVM_TCFG_PERIOD (_ULCAST_(0x1) << KVM_TCFG_PERIOD_SHIFT) +#define KVM_TCFG_EN (_ULCAST_(0x1)) + +#define KVM_CSR_TVAL 0x42 /* Timer value */ +#define KVM_CSR_CNTC 0x43 /* Timer offset */ +#define KVM_CSR_TINTCLR 0x44 /* Timer interrupt clear */ +#define KVM_CSR_GSTAT 0x50 /* Guest status */ +#define KVM_GSTAT_GID_SHIFT 16 +#define KVM_GSTAT_GID_WIDTH 8 +#define KVM_GSTAT_GID (_ULCAST_(0xff) << KVM_GSTAT_GID_SHIFT) +#define KVM_GSTAT_GIDBIT_SHIFT 4 +#define KVM_GSTAT_GIDBIT_WIDTH 6 +#define KVM_GSTAT_GIDBIT (_ULCAST_(0x3f) << KVM_GSTAT_GIDBIT_SHIFT) +#define KVM_GSTAT_PVM_SHIFT 1 +#define KVM_GSTAT_PVM (_ULCAST_(0x1) << KVM_GSTAT_PVM_SHIFT) +#define KVM_GSTAT_VM_SHIFT 0 +#define KVM_GSTAT_VM (_ULCAST_(0x1) << KVM_GSTAT_VM_SHIFT) + +#define KVM_CSR_GCFG 0x51 /* Guest config */ +#define KVM_GCFG_GPERF_SHIFT 24 +#define KVM_GCFG_GPERF_WIDTH 3 +#define KVM_GCFG_GPERF (_ULCAST_(0x7) << KVM_GCFG_GPERF_SHIFT) +#define KVM_GCFG_GCI_SHIFT 20 +#define KVM_GCFG_GCI_WIDTH 2 +#define KVM_GCFG_GCI (_ULCAST_(0x3) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_ALL (_ULCAST_(0x0) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_HIT (_ULCAST_(0x1) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_SECURE (_ULCAST_(0x2) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCIP_SHIFT 16 +#define KVM_GCFG_GCIP (_ULCAST_(0xf) << KVM_GCFG_GCIP_SHIFT) +#define KVM_GCFG_GCIP_ALL (_ULCAST_(0x1) << KVM_GCFG_GCIP_SHIFT) +#define KVM_GCFG_GCIP_HIT (_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 1)) +#define KVM_GCFG_GCIP_SECURE (_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 2)) +#define KVM_GCFG_TORU_SHIFT 15 +#define KVM_GCFG_TORU (_ULCAST_(0x1) << KVM_GCFG_TORU_SHIFT) +#define KVM_GCFG_TORUP_SHIFT 14 +#define KVM_GCFG_TORUP (_ULCAST_(0x1) << KVM_GCFG_TORUP_SHIFT) +#define KVM_GCFG_TOP_SHIFT 13 +#define KVM_GCFG_TOP (_ULCAST_(0x1) << KVM_GCFG_TOP_SHIFT) +#define KVM_GCFG_TOPP_SHIFT 12 +#define KVM_GCFG_TOPP (_ULCAST_(0x1) << KVM_GCFG_TOPP_SHIFT) +#define KVM_GCFG_TOE_SHIFT 11 +#define KVM_GCFG_TOE (_ULCAST_(0x1) << KVM_GCFG_TOE_SHIFT) +#define KVM_GCFG_TOEP_SHIFT 10 +#define KVM_GCFG_TOEP (_ULCAST_(0x1) << KVM_GCFG_TOEP_SHIFT) +#define KVM_GCFG_TIT_SHIFT 9 +#define KVM_GCFG_TIT (_ULCAST_(0x1) << KVM_GCFG_TIT_SHIFT) +#define KVM_GCFG_TITP_SHIFT 8 +#define KVM_GCFG_TITP (_ULCAST_(0x1) << KVM_GCFG_TITP_SHIFT) +#define KVM_GCFG_SIT_SHIFT 7 +#define KVM_GCFG_SIT (_ULCAST_(0x1) << KVM_GCFG_SIT_SHIFT) +#define KVM_GCFG_SITP_SHIFT 6 +#define KVM_GCFG_SITP (_ULCAST_(0x1) << KVM_GCFG_SITP_SHIFT) +#define KVM_GCFG_MATC_SHITF 4 +#define KVM_GCFG_MATC_WIDTH 2 +#define KVM_GCFG_MATC_MASK (_ULCAST_(0x3) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_GUEST (_ULCAST_(0x0) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_ROOT (_ULCAST_(0x1) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_NEST (_ULCAST_(0x2) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATP_SHITF 0 +#define KVM_GCFG_MATP_WIDTH 4 +#define KVM_GCFG_MATR_MASK (_ULCAST_(0x3) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_GUEST (_ULCAST_(0x0) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_ROOT (_ULCAST_(0x1) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_NEST (_ULCAST_(0x2) << KVM_GCFG_MATP_SHITF) + +#define KVM_CSR_GINTC 0x52 /* Guest interrupt control */ +#define KVM_CSR_GCNTC 0x53 /* Guest timer offset */ +#define KVM_CSR_LLBCTL 0x60 /* LLBit control */ +#define KVM_LLBCTL_ROLLB_SHIFT 0 +#define KVM_LLBCTL_ROLLB (_ULCAST_(1) << KVM_LLBCTL_ROLLB_SHIFT) +#define KVM_LLBCTL_WCLLB_SHIFT 1 +#define KVM_LLBCTL_WCLLB (_ULCAST_(1) << KVM_LLBCTL_WCLLB_SHIFT) +#define KVM_LLBCTL_KLO_SHIFT 2 +#define KVM_LLBCTL_KLO (_ULCAST_(1) << KVM_LLBCTL_KLO_SHIFT) + +#define KVM_CSR_IMPCTL1 0x80 /* Loongson config1 */ +#define KVM_CSR_IMPCTL2 0x81 /* Loongson config2 */ +#define KVM_CSR_GNMI 0x82 +#define KVM_CSR_TLBRENTRY 0x88 /* TLB refill exception base address */ +#define KVM_CSR_TLBRBADV 0x89 /* TLB refill badvaddr */ +#define KVM_CSR_TLBRERA 0x8a /* TLB refill ERA */ +#define KVM_CSR_TLBRSAVE 0x8b /* KScratch for TLB refill exception */ +#define KVM_CSR_TLBRELO0 0x8c /* TLB refill entrylo0 */ +#define KVM_CSR_TLBRELO1 0x8d /* TLB refill entrylo1 */ +#define KVM_CSR_TLBREHI 0x8e /* TLB refill entryhi */ +#define KVM_CSR_TLBRPRMD 0x8f /* TLB refill mode info */ +#define KVM_CSR_MERRCTL 0x90 /* ERRCTL */ +#define KVM_CSR_MERRINFO1 0x91 /* Error info1 */ +#define KVM_CSR_MERRINFO2 0x92 /* Error info2 */ +#define KVM_CSR_MERRENTRY 0x93 /* Error exception base address */ +#define KVM_CSR_MERRERA 0x94 /* Error exception PC */ +#define KVM_CSR_MERRSAVE 0x95 /* KScratch for machine error exception */ +#define KVM_CSR_CTAG 0x98 /* TagLo + TagHi */ +#define KVM_CSR_DMWIN0 0x180 /* 64 direct map win0: MEM & IF */ +#define KVM_CSR_DMWIN1 0x181 /* 64 direct map win1: MEM & IF */ +#define KVM_CSR_DMWIN2 0x182 /* 64 direct map win2: MEM */ +#define KVM_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ +#define KVM_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ +#define KVM_CSR_PERFCNTR0 0x201 /* 64 perf event 0 count value */ +#define KVM_CSR_PERFCTRL1 0x202 /* 32 perf event 1 config */ +#define KVM_CSR_PERFCNTR1 0x203 /* 64 perf event 1 count value */ +#define KVM_CSR_PERFCTRL2 0x204 /* 32 perf event 2 config */ +#define KVM_CSR_PERFCNTR2 0x205 /* 64 perf event 2 count value */ +#define KVM_CSR_PERFCTRL3 0x206 /* 32 perf event 3 config */ +#define KVM_CSR_PERFCNTR3 0x207 /* 64 perf event 3 count value */ +#define KVM_CSR_MWPC 0x300 /* data breakpoint config */ +#define KVM_CSR_MWPS 0x301 /* data breakpoint status */ +#define KVM_CSR_FWPC 0x380 /* instruction breakpoint config */ +#define KVM_CSR_FWPS 0x381 /* instruction breakpoint status */ +#define KVM_CSR_DEBUG 0x500 /* debug config */ +#define KVM_CSR_DERA 0x501 /* debug era */ +#define KVM_CSR_DESAVE 0x502 /* debug save */ + +#define KVM_IOCSR_FEATURES 0x8 +#define KVM_IOCSRF_TEMP BIT_ULL(0) +#define KVM_IOCSRF_NODECNT BIT_ULL(1) +#define KVM_IOCSRF_MSI BIT_ULL(2) +#define KVM_IOCSRF_EXTIOI BIT_ULL(3) +#define KVM_IOCSRF_CSRIPI BIT_ULL(4) +#define KVM_IOCSRF_FREQCSR BIT_ULL(5) +#define KVM_IOCSRF_FREQSCALE BIT_ULL(6) +#define KVM_IOCSRF_DVFSV1 BIT_ULL(7) +#define KVM_IOCSRF_EXTIOI_DECODE BIT_ULL(9) +#define KVM_IOCSRF_FLATMODE BIT_ULL(10) +#define KVM_IOCSRF_VM BIT_ULL(11) + +#define KVM_IOCSR_VENDOR 0x10 +#define KVM_IOCSR_CPUNAME 0x20 +#define KVM_IOCSR_NODECNT 0x408 + +#define KVM_IOCSR_MISC_FUNC 0x420 +#define KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN BIT_ULL(48) + +/* PerCore CSR, only accessable by local cores */ +#define KVM_IOCSR_IPI_STATUS 0x1000 +#define KVM_IOCSR_IPI_SEND 0x1040 +#define KVM_IOCSR_MBUF_SEND 0x1048 +#define KVM_IOCSR_EXTIOI_NODEMAP_BASE 0x14a0 +#define KVM_IOCSR_EXTIOI_IPMAP_BASE 0x14c0 +#define KVM_IOCSR_EXTIOI_EN_BASE 0x1600 +#define KVM_IOCSR_EXTIOI_BOUNCE_BASE 0x1680 +#define KVM_IOCSR_EXTIOI_ISR_BASE 0x1800 +#define KVM_IOCSR_EXTIOI_ROUTE_BASE 0x1c00 + +#ifndef __ASSEMBLY__ + +/* CSR */ +static inline u32 kvm_csr_readl(u32 reg) +{ + u32 val; + + asm volatile ( + "csrrd %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_csr_readq(u32 reg) +{ + u64 val; + + asm volatile ( + "csrrd %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline void kvm_csr_writel(u32 val, u32 reg) +{ + asm volatile ( + "csrwr %[val], %[reg] \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline void kvm_csr_writeq(u64 val, u32 reg) +{ + asm volatile ( + "csrwr %[val], %[reg] \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline u32 kvm_csr_xchgl(u32 val, u32 mask, u32 reg) +{ + asm volatile ( + "csrxchg %[val], %[mask], %[reg] \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_csr_xchgq(u64 val, u64 mask, u32 reg) +{ + asm volatile ( + "csrxchg %[val], %[mask], %[reg] \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + + +/* IOCSR */ +static inline u32 kvm_iocsr_readl(u32 reg) +{ + u32 val; + + asm volatile ( + "iocsrrd.w %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "r" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_iocsr_readq(u32 reg) +{ + u64 val; + + asm volatile ( + "iocsrrd.d %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "r" (reg) + : "memory"); + return val; +} + +static inline void kvm_iocsr_writeb(u8 val, u32 reg) +{ + asm volatile ( + "iocsrwr.b %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + +static inline void kvm_iocsr_writel(u32 val, u32 reg) +{ + asm volatile ( + "iocsrwr.w %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + +static inline void kvm_iocsr_writeq(u64 val, u32 reg) +{ + asm volatile ( + "iocsrwr.d %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + + +/* GCSR */ +static inline u64 kvm_gcsr_read(u32 reg) +{ + u64 val = 0; + + asm volatile ( + "parse_r __reg, %[val] \n" + ".word 0x5 << 24 | %[reg] << 10 | 0 << 5 | __reg \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline void kvm_gcsr_write(u64 val, u32 reg) +{ + asm volatile ( + "parse_r __reg, %[val] \n" + ".word 0x5 << 24 | %[reg] << 10 | 1 << 5 | __reg \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline u64 kvm_gcsr_xchg(u64 val, u64 mask, u32 reg) +{ + asm volatile ( + "parse_r __rd, %[val] \n" + "parse_r __rj, %[mask] \n" + ".word 0x5 << 24 | %[reg] << 10 | __rj << 5 | __rd \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + +#endif /* !__ASSEMBLY__ */ + +#define kvm_read_csr_euen() kvm_csr_readq(KVM_CSR_EUEN) +#define kvm_write_csr_euen(val) kvm_csr_writeq(val, KVM_CSR_EUEN) +#define kvm_read_csr_ecfg() kvm_csr_readq(KVM_CSR_ECFG) +#define kvm_write_csr_ecfg(val) kvm_csr_writeq(val, KVM_CSR_ECFG) +#define kvm_write_csr_perfctrl0(val) kvm_csr_writeq(val, KVM_CSR_PERFCTRL0) +#define kvm_write_csr_perfcntr0(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR0) +#define kvm_write_csr_perfctrl1(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL1) +#define kvm_write_csr_perfcntr1(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR1) +#define kvm_write_csr_perfctrl2(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL2) +#define kvm_write_csr_perfcntr2(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR2) +#define kvm_write_csr_perfctrl3(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL3) +#define kvm_write_csr_perfcntr3(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR3) +#define kvm_read_csr_impctl1() kvm_csr_readq(LOONGARCH_CSR_IMPCTL1) +#define kvm_write_csr_impctl1(val) kvm_csr_writeq(val, LOONGARCH_CSR_IMPCTL1) + + +/* Guest related CSRS */ +#define kvm_read_csr_gtlbc() kvm_csr_readq(KVM_CSR_GTLBC) +#define kvm_write_csr_gtlbc(val) kvm_csr_writeq(val, KVM_CSR_GTLBC) +#define kvm_read_csr_trgp() kvm_csr_readq(KVM_CSR_TRGP) +#define kvm_read_csr_gcfg() kvm_csr_readq(KVM_CSR_GCFG) +#define kvm_write_csr_gcfg(val) kvm_csr_writeq(val, KVM_CSR_GCFG) +#define kvm_read_csr_gstat() kvm_csr_readq(KVM_CSR_GSTAT) +#define kvm_write_csr_gstat(val) kvm_csr_writeq(val, KVM_CSR_GSTAT) +#define kvm_read_csr_gintc() kvm_csr_readq(KVM_CSR_GINTC) +#define kvm_write_csr_gintc(val) kvm_csr_writeq(val, KVM_CSR_GINTC) +#define kvm_read_csr_gcntc() kvm_csr_readq(KVM_CSR_GCNTC) +#define kvm_write_csr_gcntc(val) kvm_csr_writeq(val, KVM_CSR_GCNTC) + +/* Guest CSRS read and write */ +#define kvm_read_gcsr_crmd() kvm_gcsr_read(KVM_CSR_CRMD) +#define kvm_write_gcsr_crmd(val) kvm_gcsr_write(val, KVM_CSR_CRMD) +#define kvm_read_gcsr_prmd() kvm_gcsr_read(KVM_CSR_PRMD) +#define kvm_write_gcsr_prmd(val) kvm_gcsr_write(val, KVM_CSR_PRMD) +#define kvm_read_gcsr_euen() kvm_gcsr_read(KVM_CSR_EUEN) +#define kvm_write_gcsr_euen(val) kvm_gcsr_write(val, KVM_CSR_EUEN) +#define kvm_read_gcsr_misc() kvm_gcsr_read(KVM_CSR_MISC) +#define kvm_write_gcsr_misc(val) kvm_gcsr_write(val, KVM_CSR_MISC) +#define kvm_read_gcsr_ecfg() kvm_gcsr_read(KVM_CSR_ECFG) +#define kvm_write_gcsr_ecfg(val) kvm_gcsr_write(val, KVM_CSR_ECFG) +#define kvm_read_gcsr_estat() kvm_gcsr_read(KVM_CSR_ESTAT) +#define kvm_write_gcsr_estat(val) kvm_gcsr_write(val, KVM_CSR_ESTAT) +#define kvm_read_gcsr_era() kvm_gcsr_read(KVM_CSR_ERA) +#define kvm_write_gcsr_era(val) kvm_gcsr_write(val, KVM_CSR_ERA) +#define kvm_read_gcsr_badv() kvm_gcsr_read(KVM_CSR_BADV) +#define kvm_write_gcsr_badv(val) kvm_gcsr_write(val, KVM_CSR_BADV) +#define kvm_read_gcsr_badi() kvm_gcsr_read(KVM_CSR_BADI) +#define kvm_write_gcsr_badi(val) kvm_gcsr_write(val, KVM_CSR_BADI) +#define kvm_read_gcsr_eentry() kvm_gcsr_read(KVM_CSR_EENTRY) +#define kvm_write_gcsr_eentry(val) kvm_gcsr_write(val, KVM_CSR_EENTRY) + +#define kvm_read_gcsr_tlbidx() kvm_gcsr_read(KVM_CSR_TLBIDX) +#define kvm_write_gcsr_tlbidx(val) kvm_gcsr_write(val, KVM_CSR_TLBIDX) +#define kvm_read_gcsr_tlbhi() kvm_gcsr_read(KVM_CSR_TLBEHI) +#define kvm_write_gcsr_tlbhi(val) kvm_gcsr_write(val, KVM_CSR_TLBEHI) +#define kvm_read_gcsr_tlblo0() kvm_gcsr_read(KVM_CSR_TLBELO0) +#define kvm_write_gcsr_tlblo0(val) kvm_gcsr_write(val, KVM_CSR_TLBELO0) +#define kvm_read_gcsr_tlblo1() kvm_gcsr_read(KVM_CSR_TLBELO1) +#define kvm_write_gcsr_tlblo1(val) kvm_gcsr_write(val, KVM_CSR_TLBELO1) + +#define kvm_read_gcsr_asid() kvm_gcsr_read(KVM_CSR_ASID) +#define kvm_write_gcsr_asid(val) kvm_gcsr_write(val, KVM_CSR_ASID) +#define kvm_read_gcsr_pgdl() kvm_gcsr_read(KVM_CSR_PGDL) +#define kvm_write_gcsr_pgdl(val) kvm_gcsr_write(val, KVM_CSR_PGDL) +#define kvm_read_gcsr_pgdh() kvm_gcsr_read(KVM_CSR_PGDH) +#define kvm_write_gcsr_pgdh(val) kvm_gcsr_write(val, KVM_CSR_PGDH) +#define kvm_write_gcsr_pgd(val) kvm_gcsr_write(val, KVM_CSR_PGD) +#define kvm_read_gcsr_pgd() kvm_gcsr_read(KVM_CSR_PGD) +#define kvm_read_gcsr_pwctl0() kvm_gcsr_read(KVM_CSR_PWCTL0) +#define kvm_write_gcsr_pwctl0(val) kvm_gcsr_write(val, KVM_CSR_PWCTL0) +#define kvm_read_gcsr_pwctl1() kvm_gcsr_read(KVM_CSR_PWCTL1) +#define kvm_write_gcsr_pwctl1(val) kvm_gcsr_write(val, KVM_CSR_PWCTL1) +#define kvm_read_gcsr_stlbpgsize() kvm_gcsr_read(KVM_CSR_STLBPGSIZE) +#define kvm_write_gcsr_stlbpgsize(val) kvm_gcsr_write(val, KVM_CSR_STLBPGSIZE) +#define kvm_read_gcsr_rvacfg() kvm_gcsr_read(KVM_CSR_RVACFG) +#define kvm_write_gcsr_rvacfg(val) kvm_gcsr_write(val, KVM_CSR_RVACFG) + +#define kvm_read_gcsr_cpuid() kvm_gcsr_read(KVM_CSR_CPUID) +#define kvm_write_gcsr_cpuid(val) kvm_gcsr_write(val, KVM_CSR_CPUID) +#define kvm_read_gcsr_prcfg1() kvm_gcsr_read(KVM_CSR_PRCFG1) +#define kvm_write_gcsr_prcfg1(val) kvm_gcsr_write(val, KVM_CSR_PRCFG1) +#define kvm_read_gcsr_prcfg2() kvm_gcsr_read(KVM_CSR_PRCFG2) +#define kvm_write_gcsr_prcfg2(val) kvm_gcsr_write(val, KVM_CSR_PRCFG2) +#define kvm_read_gcsr_prcfg3() kvm_gcsr_read(KVM_CSR_PRCFG3) +#define kvm_write_gcsr_prcfg3(val) kvm_gcsr_write(val, KVM_CSR_PRCFG3) + +#define kvm_read_gcsr_kscratch0() kvm_gcsr_read(KVM_CSR_KS0) +#define kvm_write_gcsr_kscratch0(val) kvm_gcsr_write(val, KVM_CSR_KS0) +#define kvm_read_gcsr_kscratch1() kvm_gcsr_read(KVM_CSR_KS1) +#define kvm_write_gcsr_kscratch1(val) kvm_gcsr_write(val, KVM_CSR_KS1) +#define kvm_read_gcsr_kscratch2() kvm_gcsr_read(KVM_CSR_KS2) +#define kvm_write_gcsr_kscratch2(val) kvm_gcsr_write(val, KVM_CSR_KS2) +#define kvm_read_gcsr_kscratch3() kvm_gcsr_read(KVM_CSR_KS3) +#define kvm_write_gcsr_kscratch3(val) kvm_gcsr_write(val, KVM_CSR_KS3) +#define kvm_read_gcsr_kscratch4() kvm_gcsr_read(KVM_CSR_KS4) +#define kvm_write_gcsr_kscratch4(val) kvm_gcsr_write(val, KVM_CSR_KS4) +#define kvm_read_gcsr_kscratch5() kvm_gcsr_read(KVM_CSR_KS5) +#define kvm_write_gcsr_kscratch5(val) kvm_gcsr_write(val, KVM_CSR_KS5) +#define kvm_read_gcsr_kscratch6() kvm_gcsr_read(KVM_CSR_KS6) +#define kvm_write_gcsr_kscratch6(val) kvm_gcsr_write(val, KVM_CSR_KS6) +#define kvm_read_gcsr_kscratch7() kvm_gcsr_read(KVM_CSR_KS7) +#define kvm_write_gcsr_kscratch7(val) kvm_gcsr_write(val, KVM_CSR_KS7) + +#define kvm_read_gcsr_timerid() kvm_gcsr_read(KVM_CSR_TMID) +#define kvm_write_gcsr_timerid(val) kvm_gcsr_write(val, KVM_CSR_TMID) +#define kvm_read_gcsr_timercfg() kvm_gcsr_read(KVM_CSR_TCFG) +#define kvm_write_gcsr_timercfg(val) kvm_gcsr_write(val, KVM_CSR_TCFG) +#define kvm_read_gcsr_timertick() kvm_gcsr_read(KVM_CSR_TVAL) +#define kvm_write_gcsr_timertick(val) kvm_gcsr_write(val, KVM_CSR_TVAL) +#define kvm_read_gcsr_timeroffset() kvm_gcsr_read(KVM_CSR_CNTC) +#define kvm_write_gcsr_timeroffset(val) kvm_gcsr_write(val, KVM_CSR_CNTC) + +#define kvm_read_gcsr_llbctl() kvm_gcsr_read(KVM_CSR_LLBCTL) +#define kvm_write_gcsr_llbctl(val) kvm_gcsr_write(val, KVM_CSR_LLBCTL) + +#define kvm_read_gcsr_tlbrentry() kvm_gcsr_read(KVM_CSR_TLBRENTRY) +#define kvm_write_gcsr_tlbrentry(val) kvm_gcsr_write(val, KVM_CSR_TLBRENTRY) +#define kvm_read_gcsr_tlbrbadv() kvm_gcsr_read(KVM_CSR_TLBRBADV) +#define kvm_write_gcsr_tlbrbadv(val) kvm_gcsr_write(val, KVM_CSR_TLBRBADV) +#define kvm_read_gcsr_tlbrera() kvm_gcsr_read(KVM_CSR_TLBRERA) +#define kvm_write_gcsr_tlbrera(val) kvm_gcsr_write(val, KVM_CSR_TLBRERA) +#define kvm_read_gcsr_tlbrsave() kvm_gcsr_read(KVM_CSR_TLBRSAVE) +#define kvm_write_gcsr_tlbrsave(val) kvm_gcsr_write(val, KVM_CSR_TLBRSAVE) +#define kvm_read_gcsr_tlbrelo0() kvm_gcsr_read(KVM_CSR_TLBRELO0) +#define kvm_write_gcsr_tlbrelo0(val) kvm_gcsr_write(val, KVM_CSR_TLBRELO0) +#define kvm_read_gcsr_tlbrelo1() kvm_gcsr_read(KVM_CSR_TLBRELO1) +#define kvm_write_gcsr_tlbrelo1(val) kvm_gcsr_write(val, KVM_CSR_TLBRELO1) +#define kvm_read_gcsr_tlbrehi() kvm_gcsr_read(KVM_CSR_TLBREHI) +#define kvm_write_gcsr_tlbrehi(val) kvm_gcsr_write(val, KVM_CSR_TLBREHI) +#define kvm_read_gcsr_tlbrprmd() kvm_gcsr_read(KVM_CSR_TLBRPRMD) +#define kvm_write_gcsr_tlbrprmd(val) kvm_gcsr_write(val, KVM_CSR_TLBRPRMD) + +#define kvm_read_gcsr_directwin0() kvm_gcsr_read(KVM_CSR_DMWIN0) +#define kvm_write_gcsr_directwin0(val) kvm_gcsr_write(val, KVM_CSR_DMWIN0) +#define kvm_read_gcsr_directwin1() kvm_gcsr_read(KVM_CSR_DMWIN1) +#define kvm_write_gcsr_directwin1(val) kvm_gcsr_write(val, KVM_CSR_DMWIN1) +#define kvm_read_gcsr_directwin2() kvm_gcsr_read(KVM_CSR_DMWIN2) +#define kvm_write_gcsr_directwin2(val) kvm_gcsr_write(val, KVM_CSR_DMWIN2) +#define kvm_read_gcsr_directwin3() kvm_gcsr_read(KVM_CSR_DMWIN3) +#define kvm_write_gcsr_directwin3(val) kvm_gcsr_write(val, KVM_CSR_DMWIN3) + +#ifndef __ASSEMBLY__ + +static inline unsigned long +kvm_set_csr_gtlbc(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_gtlbc(); + new = res | set; + kvm_write_csr_gtlbc(new); + + return res; +} + +static inline unsigned long +kvm_set_csr_euen(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_euen(); + new = res | set; + kvm_write_csr_euen(new); + + return res; +} + +static inline unsigned long +kvm_set_csr_gintc(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_gintc(); + new = res | set; + kvm_write_csr_gintc(new); + + return res; +} + +static inline unsigned long +kvm_set_gcsr_llbctl(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_gcsr_llbctl(); + new = res | set; + kvm_write_gcsr_llbctl(new); + + return res; +} + + +static inline unsigned long +kvm_clear_csr_gtlbc(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_gtlbc(); + new = res & ~clear; + kvm_write_csr_gtlbc(new); + + return res; +} + +static inline unsigned long +kvm_clear_csr_euen(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_euen(); + new = res & ~clear; + kvm_write_csr_euen(new); + + return res; +} + +static inline unsigned long +kvm_clear_csr_gintc(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_gintc(); + new = res & ~clear; + kvm_write_csr_gintc(new); + + return res; +} + +static inline unsigned long +kvm_change_csr_gstat(unsigned long change, unsigned long val) +{ + unsigned long res, new; + + res = read_csr_gstat(); + new = res & ~change; + new |= (val & change); + write_csr_gstat(new); + + return res; +} + +static inline unsigned long +kvm_change_csr_gcfg(unsigned long change, unsigned long val) +{ + unsigned long res, new; + + res = read_csr_gcfg(); + new = res & ~change; + new |= (val & change); + write_csr_gcfg(new); + + return res; +} + + +#define kvm_set_gcsr_estat(val) \ + kvm_gcsr_xchg(val, val, KVM_CSR_ESTAT) +#define kvm_clear_gcsr_estat(val) \ + kvm_gcsr_xchg(~(val), val, KVM_CSR_ESTAT) + +#endif + + +#if (_LOONGARCH_SZLONG == 32) +#define KVM_LONG_ADD add.w +#define KVM_LONG_ADDI addi.w +#define KVM_LONG_SUB sub.w +#define KVM_LONG_L ld.w +#define KVM_LONG_S st.w +#define KVM_LONG_SLL slli.w +#define KVM_LONG_SLLV sll.w +#define KVM_LONG_SRL srli.w +#define KVM_LONG_SRLV srl.w +#define KVM_LONG_SRA srai.w +#define KVM_LONG_SRAV sra.w + +#define KVM_LONGSIZE 4 +#define KVM_LONGMASK 3 +#define KVM_LONGLOG 2 + +/* + * How to add/sub/load/store/shift pointers. + */ + +#define KVM_PTR_ADD add.w +#define KVM_PTR_ADDI addi.w +#define KVM_PTR_SUB sub.w +#define KVM_PTR_L ld.w +#define KVM_PTR_S st.w +#define KVM_PTR_LI li.w +#define KVM_PTR_SLL slli.w +#define KVM_PTR_SLLV sll.w +#define KVM_PTR_SRL srli.w +#define KVM_PTR_SRLV srl.w +#define KVM_PTR_SRA srai.w +#define KVM_PTR_SRAV sra.w + +#define KVM_PTR_SCALESHIFT 2 + +#define KVM_PTRSIZE 4 +#define KVM_PTRLOG 2 + +#endif + +#if (_LOONGARCH_SZLONG == 64) +#define KVM_LONG_ADD add.d +#define KVM_LONG_ADDI addi.d +#define KVM_LONG_SUB sub.d +#define KVM_LONG_L ld.d +#define KVM_LONG_S st.d +#define KVM_LONG_SLL slli.d +#define KVM_LONG_SLLV sll.d +#define KVM_LONG_SRL srli.d +#define KVM_LONG_SRLV srl.d +#define KVM_LONG_SRA sra.w +#define KVM_LONG_SRAV sra.d + +#define KVM_LONGSIZE 8 +#define KVM_LONGMASK 7 +#define KVM_LONGLOG 3 + +/* + * How to add/sub/load/store/shift pointers. + */ + +#define KVM_PTR_ADD add.d +#define KVM_PTR_ADDI addi.d +#define KVM_PTR_SUB sub.d +#define KVM_PTR_L ld.d +#define KVM_PTR_S st.d +#define KVM_PTR_LI li.d +#define KVM_PTR_SLL slli.d +#define KVM_PTR_SLLV sll.d +#define KVM_PTR_SRL srli.d +#define KVM_PTR_SRLV srl.d +#define KVM_PTR_SRA srai.d +#define KVM_PTR_SRAV sra.d + +#define KVM_PTR_SCALESHIFT 3 + +#define KVM_PTRSIZE 8 +#define KVM_PTRLOG 3 +#endif + +#endif /* __LOONGARCH_KVM_COMPAT_H__ */ diff --git a/arch/loongarch/kvm/kvmcpu.h b/arch/loongarch/kvm/kvmcpu.h new file mode 100644 index 0000000000000000000000000000000000000000..8cb411ba9f17662a5a7d2ed4824ce05a55d2f356 --- /dev/null +++ b/arch/loongarch/kvm/kvmcpu.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __ASM_LOONGARCH_KVMCPU_H__ +#define __ASM_LOONGARCH_KVMCPU_H__ + +#include +#include + +#define LARCH_INT_SIP0 0 +#define LARCH_INT_SIP1 1 +#define LARCH_INT_IP0 2 +#define LARCH_INT_IP1 3 +#define LARCH_INT_IP2 4 +#define LARCH_INT_IP3 5 +#define LARCH_INT_IP4 6 +#define LARCH_INT_IP5 7 +#define LARCH_INT_IP6 8 +#define LARCH_INT_IP7 9 +#define LARCH_INT_PMU 10 +#define LARCH_INT_TIMER 11 +#define LARCH_INT_IPI 12 +#define LOONGARCH_EXC_MAX (LARCH_INT_IPI + 1) +#define LOONGARCH_EXC_IPNUM (LOONGARCH_EXC_MAX) + +/* Controlled by 0x5 guest exst */ +#define CPU_SIP0 (_ULCAST_(1)) +#define CPU_SIP1 (_ULCAST_(1) << 1) +#define CPU_PMU (_ULCAST_(1) << 10) +#define CPU_TIMER (_ULCAST_(1) << 11) +#define CPU_IPI (_ULCAST_(1) << 12) + +/* Controlled by 0x52 guest exception VIP + * aligned to exst bit 5~12 + */ +#define CPU_IP0 (_ULCAST_(1)) +#define CPU_IP1 (_ULCAST_(1) << 1) +#define CPU_IP2 (_ULCAST_(1) << 2) +#define CPU_IP3 (_ULCAST_(1) << 3) +#define CPU_IP4 (_ULCAST_(1) << 4) +#define CPU_IP5 (_ULCAST_(1) << 5) +#define CPU_IP6 (_ULCAST_(1) << 6) +#define CPU_IP7 (_ULCAST_(1) << 7) + +#define MNSEC_PER_SEC (NSEC_PER_SEC >> 20) + +/* KVM_IRQ_LINE irq field index values */ +#define KVM_LOONGSON_IRQ_TYPE_SHIFT 24 +#define KVM_LOONGSON_IRQ_TYPE_MASK 0xff +#define KVM_LOONGSON_IRQ_VCPU_SHIFT 16 +#define KVM_LOONGSON_IRQ_VCPU_MASK 0xff +#define KVM_LOONGSON_IRQ_NUM_SHIFT 0 +#define KVM_LOONGSON_IRQ_NUM_MASK 0xffff + +/* irq_type field */ +#define KVM_LOONGSON_IRQ_TYPE_CPU_IP 0 +#define KVM_LOONGSON_IRQ_TYPE_CPU_IO 1 +#define KVM_LOONGSON_IRQ_TYPE_HT 2 +#define KVM_LOONGSON_IRQ_TYPE_MSI 3 +#define KVM_LOONGSON_IRQ_TYPE_IOAPIC 4 +#define KVM_LOONGSON_IRQ_TYPE_ROUTE 5 + +/* out-of-kernel GIC cpu interrupt injection irq_number field */ +#define KVM_LOONGSON_IRQ_CPU_IRQ 0 +#define KVM_LOONGSON_IRQ_CPU_FIQ 1 +#define KVM_LOONGSON_CPU_IP_NUM 8 + +typedef union loongarch_instruction larch_inst; +typedef int (*exit_handle_fn)(struct kvm_vcpu *); + +int _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst); +int _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst); +int _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run); +int _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run); +int _kvm_emu_idle(struct kvm_vcpu *vcpu); +int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu); +int _kvm_pending_timer(struct kvm_vcpu *vcpu); +int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault); +void _kvm_deliver_intr(struct kvm_vcpu *vcpu); +void irqchip_debug_init(struct kvm *kvm); +void irqchip_debug_destroy(struct kvm *kvm); + +void kvm_own_fpu(struct kvm_vcpu *vcpu); +void kvm_own_lsx(struct kvm_vcpu *vcpu); +void kvm_lose_fpu(struct kvm_vcpu *vcpu); +void kvm_own_lasx(struct kvm_vcpu *vcpu); +void kvm_own_lbt(struct kvm_vcpu *vcpu); +void kvm_restore_lbt(struct kvm_vcpu *cpu); +void kvm_save_lbt(struct kvm_vcpu *cpu); +void kvm_save_fpu(struct kvm_vcpu *cpu); +void kvm_restore_fpu(struct kvm_vcpu *cpu); +void kvm_save_lsx(struct kvm_vcpu *cpu); +void kvm_restore_lsx(struct kvm_vcpu *cpu); +void kvm_restore_lsx_upper(struct kvm_vcpu *cpu); +void kvm_save_lasx(struct kvm_vcpu *cpu); +void kvm_restore_lasx(struct kvm_vcpu *cpu); +void kvm_restore_lasx_upper(struct kvm_vcpu *cpu); + +void kvm_lose_hw_perf(struct kvm_vcpu *vcpu); +void kvm_restore_hw_perf(struct kvm_vcpu *vcpu); + +void kvm_acquire_timer(struct kvm_vcpu *vcpu); +void kvm_reset_timer(struct kvm_vcpu *vcpu); +enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu); +void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); +void kvm_restore_timer(struct kvm_vcpu *vcpu); +void kvm_save_timer(struct kvm_vcpu *vcpu); + +/* + * Loongarch KVM guest interrupt handling. + */ +static inline void _kvm_queue_irq(struct kvm_vcpu *vcpu, unsigned int irq) +{ + set_bit(irq, &vcpu->arch.irq_pending); + clear_bit(irq, &vcpu->arch.irq_clear); +} + +static inline void _kvm_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int irq) +{ + clear_bit(irq, &vcpu->arch.irq_pending); + set_bit(irq, &vcpu->arch.irq_clear); +} + +#endif /* __ASM_LOONGARCH_KVMCPU_H__ */ diff --git a/arch/loongarch/kvm/kvmcsr.h b/arch/loongarch/kvm/kvmcsr.h new file mode 100644 index 0000000000000000000000000000000000000000..a93ed920402f3e2132309abc9438854b7f7354ee --- /dev/null +++ b/arch/loongarch/kvm/kvmcsr.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGARCH_KVM_CSR_H__ +#define __LOONGARCH_KVM_CSR_H__ +#include +#include +#include +#include +#include "kvmcpu.h" + +#define kvm_read_hw_gcsr(id) gcsr_read(id) +#define kvm_write_hw_gcsr(csr, id, val) gcsr_write(val, id) + +int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force); +int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force); +unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid); +void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val); +void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long csr_mask, unsigned long val); +int _kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu); + +static inline void kvm_save_hw_gcsr(struct loongarch_csrs *csr, int gid) +{ + csr->csrs[gid] = gcsr_read(gid); +} + +static inline void kvm_restore_hw_gcsr(struct loongarch_csrs *csr, int gid) +{ + gcsr_write(csr->csrs[gid], gid); +} + +static inline unsigned long kvm_read_sw_gcsr(struct loongarch_csrs *csr, int gid) +{ + return csr->csrs[gid]; +} + +static inline void kvm_write_sw_gcsr(struct loongarch_csrs *csr, int gid, unsigned long val) +{ + csr->csrs[gid] = val; +} + +static inline void kvm_set_sw_gcsr(struct loongarch_csrs *csr, int gid, unsigned long val) +{ + csr->csrs[gid] |= val; +} + +static inline void kvm_change_sw_gcsr(struct loongarch_csrs *csr, int gid, unsigned mask, + unsigned long val) +{ + unsigned long _mask = mask; + csr->csrs[gid] &= ~_mask; + csr->csrs[gid] |= val & _mask; +} + + +#define GET_HW_GCSR(id, csrid, v) \ + do { \ + if (csrid == id) { \ + *v = (long)kvm_read_hw_gcsr(csrid); \ + return 0; \ + } \ + } while (0) + +#define GET_SW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + *v = kvm_read_sw_gcsr(csr, id); \ + return 0; \ + } \ + } while (0) + +#define SET_HW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + kvm_write_hw_gcsr(csr, csrid, *v); \ + return 0; \ + } \ + } while (0) + +#define SET_SW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + kvm_write_sw_gcsr(csr, csrid, *v); \ + return 0; \ + } \ + } while (0) + +int _kvm_init_iocsr(struct kvm *kvm); +int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); +int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); + +#define KVM_PMU_PLV_ENABLE (CSR_PERFCTRL_PLV0 | \ + CSR_PERFCTRL_PLV1 | \ + CSR_PERFCTRL_PLV2 | \ + CSR_PERFCTRL_PLV3) + +#define CASE_WRITE_HW_PMU(vcpu, csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + if (v & KVM_PMU_PLV_ENABLE) { \ + write_csr_gcfg(read_csr_gcfg() | CSR_GCFG_GPERF); \ + kvm_write_hw_gcsr(csr, csrid, v | CSR_PERFCTRL_GMOD); \ + vcpu->arch.aux_inuse |= KVM_LARCH_PERF; \ + return ; \ + } else { \ + kvm_write_sw_gcsr(csr, csrid, v); \ + return; \ + } \ + } \ + } while (0) + +#endif /* __LOONGARCH_KVM_CSR_H__ */ diff --git a/arch/loongarch/kvm/loongarch.c b/arch/loongarch/kvm/loongarch.c new file mode 100644 index 0000000000000000000000000000000000000000..104f0e31d7485e06d5e68e50dcf3aec6d3c0c890 --- /dev/null +++ b/arch/loongarch/kvm/loongarch.c @@ -0,0 +1,2208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include +#include +#include + +#include "intc/ls3a_ipi.h" +#include "intc/ls7a_irq.h" +#include "intc/ls3a_ext_irq.h" +#include "kvm_compat.h" +#include "kvmcsr.h" +#include "ls_irq.h" + +/* + * Define loongarch kvm version. + * Add version number when qemu/kvm interface changed + */ +#define KVM_LOONGARCH_VERSION 1 +#define CREATE_TRACE_POINTS +#include "trace.h" + +struct kvm_stats_debugfs_item vcpu_debugfs_entries[] = { + VCPU_STAT("idle", idle_exits), + VCPU_STAT("signal", signal_exits), + VCPU_STAT("interrupt", int_exits), + VCPU_STAT("tlbmiss_ld", excep_exits[EXCCODE_TLBL]), + VCPU_STAT("tlbmiss_st", excep_exits[EXCCODE_TLBS]), + VCPU_STAT("tlb_ifetch", excep_exits[EXCCODE_TLBI]), + VCPU_STAT("tlbmod", excep_exits[EXCCODE_TLBM]), + VCPU_STAT("tlbri", excep_exits[EXCCODE_TLBNR]), + VCPU_STAT("tlbxi", excep_exits[EXCCODE_TLBNX]), + VCPU_STAT("fp_disabled", excep_exits[EXCCODE_FPDIS]), + VCPU_STAT("lsx_disabled", excep_exits[EXCCODE_LSXDIS]), + VCPU_STAT("lasx_disabled", excep_exits[EXCCODE_LASXDIS]), + VCPU_STAT("fpe", excep_exits[EXCCODE_FPE]), + VCPU_STAT("watch", excep_exits[EXCCODE_WATCH]), + VCPU_STAT("gspr", excep_exits[EXCCODE_GSPR]), + VCPU_STAT("vz_gsfc", excep_exits[EXCCODE_GCM]), + VCPU_STAT("vz_hc", excep_exits[EXCCODE_HVC]), + + VCPU_STAT("rdcsr_cpu_feature", rdcsr_cpu_feature_exits), + VCPU_STAT("rdcsr_misc_func", rdcsr_misc_func_exits), + VCPU_STAT("rdcsr_ipi_access", rdcsr_ipi_access_exits), + VCPU_STAT("cpucfg", cpucfg_exits), + VCPU_STAT("huge_dec", huge_dec_exits), + VCPU_STAT("huge_thp", huge_thp_exits), + VCPU_STAT("huge_adj", huge_adjust_exits), + VCPU_STAT("huge_set", huge_set_exits), + VCPU_STAT("huge_merg", huge_merge_exits), + + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), + {NULL} +}; + +struct kvm_stats_debugfs_item debugfs_entries[] = { + VM_STAT("remote_tlb_flush", remote_tlb_flush), + VM_STAT("pip_read_exits", pip_read_exits), + VM_STAT("pip_write_exits", pip_write_exits), + VM_STAT("vm_ioctl_irq_line", vm_ioctl_irq_line), + VM_STAT("ls7a_ioapic_update", ls7a_ioapic_update), + VM_STAT("ls7a_ioapic_set_irq", ls7a_ioapic_set_irq), + VM_STAT("ls7a_msi_irq", ls7a_msi_irq), + VM_STAT("ioapic_reg_write", ioapic_reg_write), + VM_STAT("ioapic_reg_read", ioapic_reg_read), + VM_STAT("set_ls7a_ioapic", set_ls7a_ioapic), + VM_STAT("get_ls7a_ioapic", get_ls7a_ioapic), + VM_STAT("set_ls3a_ext_irq", set_ls3a_ext_irq), + VM_STAT("get_ls3a_ext_irq", get_ls3a_ext_irq), + VM_STAT("ls3a_ext_irq", trigger_ls3a_ext_irq), + {NULL} +}; + +static int lvcpu_stat_get(void *address, u64 *val) +{ + *val = *(u64 *)address; + return 0; +} +DEFINE_SIMPLE_ATTRIBUTE(lvcpu_stat_fops, lvcpu_stat_get, NULL, "%llu\n"); + +static int vcpu_pid_get(void *arg, u64 *val) +{ + struct kvm_vcpu *vcpu = (struct kvm_vcpu *)arg; + if (vcpu) + *val = pid_vnr(vcpu->pid); + return 0; +} +DEFINE_SIMPLE_ATTRIBUTE(vcpu_pid_fops, vcpu_pid_get, NULL, "%llu\n"); + +void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry) +{ + struct kvm_stats_debugfs_item *p; + debugfs_create_file("pid", 0444, debugfs_dentry, vcpu, &vcpu_pid_fops); + for (p = vcpu_debugfs_entries; p->name && p->kind == KVM_STAT_VCPU; ++p) { + debugfs_create_file(p->name, 0444, debugfs_dentry, + (void *)vcpu + p->offset, &lvcpu_stat_fops); + } +} + +bool kvm_trace_guest_mode_change; +static struct kvm_context __percpu *vmcs; + +int kvm_guest_mode_change_trace_reg(void) +{ + kvm_trace_guest_mode_change = 1; + return 0; +} + +void kvm_guest_mode_change_trace_unreg(void) +{ + kvm_trace_guest_mode_change = 0; +} + +static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.pv.pv_unhalted) + return true; + + return false; +} + +/* + * XXXKYMA: We are simulatoring a processor that has the WII bit set in + * Config7, so we are "runnable" if interrupts are pending + */ +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return !!(vcpu->arch.irq_pending) || kvm_vcpu_has_events(vcpu); +} + +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return false; +} + +int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; +} + +#ifdef CONFIG_PARAVIRT +void kvm_update_stolen_time(struct kvm_vcpu *vcpu) +{ + struct kvm_host_map map; + struct kvm_steal_time *st; + int ret = 0; + + if (vcpu->arch.st.guest_addr == 0) + return; + + ret = kvm_map_gfn(vcpu, vcpu->arch.st.guest_addr >> PAGE_SHIFT, + &map, &vcpu->arch.st.cache, false); + if (ret) { + kvm_info("%s ret:%d\n", __func__, ret); + return; + } + st = map.hva + offset_in_page(vcpu->arch.st.guest_addr); + if (st->version & 1) + st->version += 1; /* first time write, random junk */ + st->version += 1; + smp_wmb(); + st->steal += current->sched_info.run_delay - + vcpu->arch.st.last_steal; + vcpu->arch.st.last_steal = current->sched_info.run_delay; + smp_wmb(); + st->version += 1; + + kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false); +} + +bool _kvm_pvtime_supported(void) +{ + return !!sched_info_on(); +} + +int _kvm_pvtime_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *user = (u64 __user *)attr->addr; + struct kvm *kvm = vcpu->kvm; + u64 ipa; + int ret = 0; + int idx; + + if (!_kvm_pvtime_supported() || + attr->attr != KVM_LARCH_VCPU_PVTIME_IPA) + return -ENXIO; + + if (get_user(ipa, user)) + return -EFAULT; + if (!IS_ALIGNED(ipa, 64)) + return -EINVAL; + + /* Check the address is in a valid memslot */ + idx = srcu_read_lock(&kvm->srcu); + if (kvm_is_error_hva(gfn_to_hva(kvm, ipa >> PAGE_SHIFT))) + ret = -EINVAL; + srcu_read_unlock(&kvm->srcu, idx); + + if (!ret) + vcpu->arch.st.guest_addr = ipa; + + return ret; +} + +int _kvm_pvtime_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *user = (u64 __user *)attr->addr; + u64 ipa; + + if (!_kvm_pvtime_supported() || + attr->attr != KVM_LARCH_VCPU_PVTIME_IPA) + return -ENXIO; + + ipa = vcpu->arch.st.guest_addr; + + if (put_user(ipa, user)) + return -EFAULT; + + return 0; +} + +int _kvm_pvtime_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_LARCH_VCPU_PVTIME_IPA: + if (_kvm_pvtime_supported()) + return 0; + } + + return -ENXIO; +} + +static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) +{ + struct kvm_host_map map; + struct kvm_steal_time *st; + int ret = 0; + + if (vcpu->arch.st.guest_addr == 0) + return; + + ret = kvm_map_gfn(vcpu, vcpu->arch.st.guest_addr >> PAGE_SHIFT, + &map, &vcpu->arch.st.cache, false); + if (ret) { + kvm_info("%s ret:%d\n", __func__, ret); + return; + } + st = map.hva + offset_in_page(vcpu->arch.st.guest_addr); + if (st->version & 1) + st->version += 1; /* first time write, random junk */ + st->version += 1; + smp_wmb(); + st->preempted = KVM_VCPU_PREEMPTED; + smp_wmb(); + st->version += 1; + + kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false); +} + +static void kvm_steal_time_clear_preempted(struct kvm_vcpu *vcpu) +{ + struct kvm_host_map map; + struct kvm_steal_time *st; + int ret = 0; + + if (vcpu->arch.st.guest_addr == 0) + return; + + ret = kvm_map_gfn(vcpu, vcpu->arch.st.guest_addr >> PAGE_SHIFT, + &map, &vcpu->arch.st.cache, false); + if (ret) { + kvm_info("%s ret:%d\n", __func__, ret); + return; + } + st = map.hva + offset_in_page(vcpu->arch.st.guest_addr); + if (st->version & 1) + st->version += 1; /* first time write, random junk */ + st->version += 1; + smp_wmb(); + st->preempted = 0; + smp_wmb(); + st->version += 1; + + kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false); +} +#endif + +int kvm_arch_hardware_enable(void) +{ + unsigned long gcfg = 0; + + /* First init gtlbc, gcfg, gstat, gintc. All guest use the same config */ + kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI); + kvm_write_csr_gcfg(0); + kvm_write_csr_gstat(0); + kvm_write_csr_gintc(0); + + /* + * Enable virtualization features granting guest direct control of + * certain features: + * GCI=2: Trap on init or unimplement cache instruction. + * TORU=0: Trap on Root Unimplement. + * CACTRL=1: Root control cache. + * TOP=0: Trap on Previlege. + * TOE=0: Trap on Exception. + * TIT=0: Trap on Timer. + */ + if (cpu_has_gcip_all) + gcfg |= KVM_GCFG_GCI_SECURE; + if (cpu_has_matc_root) + gcfg |= KVM_GCFG_MATC_ROOT; + + gcfg |= KVM_GCFG_TIT; + kvm_write_csr_gcfg(gcfg); + + kvm_flush_tlb_all(); + + /* Enable using TGID */ + kvm_set_csr_gtlbc(KVM_GTLBC_USETGID); + kvm_debug("gtlbc:%llx gintc:%llx gstat:%llx gcfg:%llx", + kvm_read_csr_gtlbc(), kvm_read_csr_gintc(), + kvm_read_csr_gstat(), kvm_read_csr_gcfg()); + return 0; +} + +void kvm_arch_hardware_disable(void) +{ + kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI); + kvm_write_csr_gcfg(0); + kvm_write_csr_gstat(0); + kvm_write_csr_gintc(0); + + /* Flush any remaining guest TLB entries */ + kvm_flush_tlb_all(); +} + +int kvm_arch_hardware_setup(void *opaque) +{ + return 0; +} + +int kvm_arch_check_processor_compat(void *rtn) +{ + return 0; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + /* Allocate page table to map GPA -> RPA */ + kvm->arch.gpa_mm.pgd = kvm_pgd_alloc(); + if (!kvm->arch.gpa_mm.pgd) + return -ENOMEM; + + kvm->arch.cpucfg_lasx = (read_cpucfg(LOONGARCH_CPUCFG2) & + CPUCFG2_LASX); + + _kvm_init_iocsr(kvm); + kvm->arch.vmcs = vmcs; + + return 0; +} + +static void kvm_free_vcpus(struct kvm *kvm) +{ + unsigned int i; + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + kvm_vcpu_destroy(vcpu); + } + + mutex_lock(&kvm->lock); + + for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) + kvm->vcpus[i] = NULL; + + atomic_set(&kvm->online_vcpus, 0); + + mutex_unlock(&kvm->lock); +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_ls3a_ipi(kvm); + kvm_destroy_ls7a_ioapic(kvm); + kvm_destroy_ls3a_ext_irq(kvm); + kvm_free_vcpus(kvm); + _kvm_destroy_mm(kvm); +} + +long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned long npages) +{ + return 0; +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + return 0; +} + +static void _kvm_new_vpid(unsigned long cpu, struct kvm_vcpu *vcpu) +{ + struct kvm_context *context; + unsigned long vpid; + + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + vpid = context->vpid_cache; + if (!(++vpid & context->gid_mask)) { + if (!vpid) /* fix version if needed */ + vpid = context->gid_fisrt_ver; + + ++vpid; /* vpid 0 reserved for root */ + + /* start new vpid cycle */ + kvm_flush_tlb_all(); + } + + context->vpid_cache = vpid; + vcpu->arch.vpid[cpu] = vpid; +} + +/* Returns 1 if the guest TLB may be clobbered */ +static int _kvm_check_requests(struct kvm_vcpu *vcpu, int cpu) +{ + int ret = 0; + int i; + + if (!kvm_request_pending(vcpu)) + return 0; + + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { + /* Drop all vpids for this VCPU */ + for_each_possible_cpu(i) + vcpu->arch.vpid[i] = 0; + /* This will clobber guest TLB contents too */ + ret = 1; + } + + return ret; +} + +static void _kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu) +{ + struct kvm_context *context; + bool migrated; + unsigned int gstinfo_gidmask, gstinfo_gid = 0; + + /* + * Are we entering guest context on a different CPU to last time? + * If so, the VCPU's guest TLB state on this CPU may be stale. + */ + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + migrated = (vcpu->arch.last_exec_cpu != cpu); + vcpu->arch.last_exec_cpu = cpu; + + /* + * Check if our vpid is of an older version and thus invalid. + * + * We also discard the stored vpid if we've executed on + * another CPU, as the guest mappings may have changed without + * hypervisor knowledge. + */ + gstinfo_gidmask = context->gid_mask << KVM_GSTAT_GID_SHIFT; + if (migrated || + (vcpu->arch.vpid[cpu] ^ context->vpid_cache) & + context->gid_ver_mask) { + _kvm_new_vpid(cpu, vcpu); + trace_kvm_vpid_change(vcpu, vcpu->arch.vpid[cpu]); + } + gstinfo_gid = (vcpu->arch.vpid[cpu] & context->gid_mask) << + KVM_GSTAT_GID_SHIFT; + + /* Restore GSTAT(0x50).vpid */ + kvm_change_csr_gstat(gstinfo_gidmask, gstinfo_gid); +} + +/* + * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV) + */ +static int _kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) +{ + unsigned long exst = vcpu->arch.host_estat; + u32 intr = exst & 0x1fff; /* ignore NMI */ + u32 exccode = (exst & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT; + u32 __user *opc = (u32 __user *) vcpu->arch.pc; + int ret = RESUME_GUEST, cpu; + + vcpu->mode = OUTSIDE_GUEST_MODE; + + /* Set a default exit reason */ + run->exit_reason = KVM_EXIT_UNKNOWN; + run->ready_for_interrupt_injection = 1; + + /* + * Set the appropriate status bits based on host CPU features, + * before we hit the scheduler + */ + + local_irq_enable(); + + kvm_debug("%s: exst: %lx, PC: %p, kvm_run: %p, kvm_vcpu: %p\n", + __func__, exst, opc, run, vcpu); + trace_kvm_exit(vcpu, exccode); + if (exccode) { + vcpu->stat.excep_exits[exccode]++; + ret = _kvm_handle_fault(vcpu, exccode); + } else { + WARN(!intr, "suspicious vm exiting"); + ++vcpu->stat.int_exits; + + if (need_resched()) + cond_resched(); + + ret = RESUME_GUEST; + } + +#ifdef CONFIG_PARAVIRT + if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) + kvm_update_stolen_time(vcpu); +#endif + + cond_resched(); + + local_irq_disable(); + + if (ret == RESUME_GUEST) { + /* Only check for signals if not already exiting to userspace */ + if (signal_pending(current)) { + run->exit_reason = KVM_EXIT_INTR; + ret = (-EINTR << 2) | RESUME_HOST; + ++vcpu->stat.signal_exits; + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL); + return ret; + } + + trace_kvm_reenter(vcpu); + + kvm_acquire_timer(vcpu); + _kvm_deliver_intr(vcpu); + + /* + * Make sure the read of VCPU requests in vcpu_reenter() + * callback is not reordered ahead of the write to vcpu->mode, + * or we could miss a TLB flush request while the requester sees + * the VCPU as outside of guest mode and not needing an IPI. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + cpu = smp_processor_id(); + _kvm_check_requests(vcpu, cpu); + _kvm_update_vmid(vcpu, cpu); + } + + return ret; +} + +/* low level hrtimer wake routine */ +static enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer) +{ + struct kvm_vcpu *vcpu; + + vcpu = container_of(timer, struct kvm_vcpu, arch.swtimer); + + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); + + rcuwait_wake_up(&vcpu->wait); + + return kvm_count_timeout(vcpu); +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + int i; + unsigned long timer_hz; + struct loongarch_csrs *csr = vcpu->arch.csr; + + for_each_possible_cpu(i) + vcpu->arch.vpid[i] = 0; + + hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); + vcpu->arch.swtimer.function = kvm_swtimer_wakeup; + vcpu->arch.fpu_enabled = true; + vcpu->arch.lsx_enabled = true; + + vcpu->kvm->arch.online_vcpus = vcpu->vcpu_id + 1; + + vcpu->arch.host_eentry = kvm_csr_readq(KVM_CSR_EENTRY); + vcpu->arch.guest_eentry = (unsigned long)kvm_exception_entry; + vcpu->arch.vcpu_run = kvm_enter_guest; + vcpu->arch.handle_exit = _kvm_handle_exit; + vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); + /* + * kvm all exceptions share one exception entry, and host <-> guest switch + * also switch excfg.VS field, keep host excfg.VS info here + */ + vcpu->arch.host_ecfg = (kvm_read_csr_ecfg() & KVM_ECFG_VS); + + if (!vcpu->arch.csr) + return -ENOMEM; + + /* Init */ + vcpu->arch.last_sched_cpu = -1; + vcpu->arch.last_exec_cpu = -1; + + /* + * Initialize guest register state to valid architectural reset state. + */ + timer_hz = calc_const_freq(); + kvm_init_timer(vcpu, timer_hz); + + /* Set Initialize mode for GUEST */ + kvm_write_sw_gcsr(csr, KVM_CSR_CRMD, KVM_CRMD_DA); + + /* Set cpuid */ + kvm_write_sw_gcsr(csr, KVM_CSR_TMID, vcpu->vcpu_id); + + /* start with no pending virtual guest interrupts */ + csr->csrs[KVM_CSR_GINTC] = 0; + + return 0; +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + int cpu; + struct kvm_context *context; + struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache; + + hrtimer_cancel(&vcpu->arch.swtimer); + kvm_mmu_free_memory_caches(vcpu); + if (vcpu->arch.st.guest_addr) + kvm_release_pfn(cache->pfn, cache->dirty, cache); + kfree(vcpu->arch.csr); + + /* + * If the VCPU is freed and reused as another VCPU, we don't want the + * matching pointer wrongly hanging around in last_vcpu. + */ + for_each_possible_cpu(cpu) { + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + if (context->last_vcpu == vcpu) + context->last_vcpu = NULL; + } +} + +#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ + KVM_GUESTDBG_USE_SW_BP | KVM_GUESTDBG_SINGLESTEP) +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + int ret = 0; + + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { + ret = -EINVAL; + goto out; + } + if (dbg->control & KVM_GUESTDBG_ENABLE) { + vcpu->guest_debug = dbg->control; + /* No hardware breakpoint */ + } else { + vcpu->guest_debug = 0; + } +out: + return ret; +} + +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + int r = -EINTR; + int cpu; + + vcpu_load(vcpu); + + kvm_sigset_activate(vcpu); + + if (vcpu->mmio_needed) { + if (!vcpu->mmio_is_write) + _kvm_complete_mmio_read(vcpu, vcpu->run); + vcpu->mmio_needed = 0; + } else if (vcpu->arch.is_hypcall) { + /* set return value for hypercall v0 register */ + vcpu->arch.gprs[REG_A0] = vcpu->run->hypercall.ret; + vcpu->arch.is_hypcall = 0; + } + + if (vcpu->run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) { + if (!vcpu->run->iocsr_io.is_write) + _kvm_complete_iocsr_read(vcpu, vcpu->run); + } + + /* clear exit_reason */ + vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; + if (vcpu->run->immediate_exit) + goto out; + + lose_fpu(1); + lose_lbt(1); + +#ifdef CONFIG_PARAVIRT + if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) + kvm_update_stolen_time(vcpu); +#endif + local_irq_disable(); + guest_enter_irqoff(); + trace_kvm_enter(vcpu); + + /* + * Make sure the read of VCPU requests in vcpu_run() callback is not + * reordered ahead of the write to vcpu->mode, or we could miss a TLB + * flush request while the requester sees the VCPU as outside of guest + * mode and not needing an IPI. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + cpu = smp_processor_id(); + kvm_acquire_timer(vcpu); + /* Check if we have any exceptions/interrupts pending */ + _kvm_deliver_intr(vcpu); + + _kvm_check_requests(vcpu, cpu); + _kvm_update_vmid(vcpu, cpu); + r = kvm_enter_guest(vcpu->run, vcpu); + + trace_kvm_out(vcpu); + guest_exit_irqoff(); + local_irq_enable(); + +out: + kvm_sigset_deactivate(vcpu); + + vcpu_put(vcpu); + return r; +} + +int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq) +{ + int intr = (int)irq->irq; + + if (intr < 0) { + _kvm_dequeue_irq(vcpu, -intr); + return 0; + } + + _kvm_queue_irq(vcpu, intr); + kvm_vcpu_kick(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +/** + * kvm_migrate_count() - Migrate timer. + * @vcpu: Virtual CPU. + * + * Migrate hrtimer to the current CPU by cancelling and restarting it + * if it was running prior to being cancelled. + * + * Must be called when the VCPU is migrated to a different CPU to ensure that + * timer expiry during guest execution interrupts the guest and causes the + * interrupt to be delivered in a timely manner. + */ +static void kvm_migrate_count(struct kvm_vcpu *vcpu) +{ + if (hrtimer_cancel(&vcpu->arch.swtimer)) + hrtimer_restart(&vcpu->arch.swtimer); +} + +static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + struct kvm_context *context; + struct loongarch_csrs *csr = vcpu->arch.csr; + bool migrated, all; + + /* + * Have we migrated to a different CPU? + * If so, any old guest TLB state may be stale. + */ + migrated = (vcpu->arch.last_sched_cpu != cpu); + + /* + * Was this the last VCPU to run on this CPU? + * If not, any old guest state from this VCPU will have been clobbered. + */ + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + all = migrated || (context->last_vcpu != vcpu); + context->last_vcpu = vcpu; + + /* + * Restore timer state regardless + */ + kvm_restore_timer(vcpu); + + /* Control guest page CCA attribute */ + kvm_change_csr_gcfg(KVM_GCFG_MATC_MASK, KVM_GCFG_MATC_ROOT); + /* Restore hardware perf csr */ + kvm_restore_hw_perf(vcpu); + +#ifdef CONFIG_PARAVIRT + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +#endif + /* Don't bother restoring registers multiple times unless necessary */ + if (!all) + return 0; + + kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset); + /* + * Restore guest CSR registers + */ + kvm_restore_hw_gcsr(csr, KVM_CSR_CRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_PRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_EUEN); + kvm_restore_hw_gcsr(csr, KVM_CSR_MISC); + kvm_restore_hw_gcsr(csr, KVM_CSR_ECFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_ERA); + kvm_restore_hw_gcsr(csr, KVM_CSR_BADV); + kvm_restore_hw_gcsr(csr, KVM_CSR_BADI); + kvm_restore_hw_gcsr(csr, KVM_CSR_EENTRY); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBIDX); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBEHI); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO0); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO1); + kvm_restore_hw_gcsr(csr, KVM_CSR_ASID); + kvm_restore_hw_gcsr(csr, KVM_CSR_PGDL); + kvm_restore_hw_gcsr(csr, KVM_CSR_PGDH); + kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL1); + kvm_restore_hw_gcsr(csr, KVM_CSR_STLBPGSIZE); + kvm_restore_hw_gcsr(csr, KVM_CSR_RVACFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_CPUID); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS0); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS1); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS2); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS3); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS4); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS5); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS6); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS7); + kvm_restore_hw_gcsr(csr, KVM_CSR_TMID); + kvm_restore_hw_gcsr(csr, KVM_CSR_CNTC); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRENTRY); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRBADV); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRERA); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRSAVE); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO0); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO1); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBREHI); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRPRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN0); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN1); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN2); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN3); + kvm_restore_hw_gcsr(csr, KVM_CSR_LLBCTL); + + /* restore Root.Guestexcept from unused Guest guestexcept register */ + kvm_write_csr_gintc(csr->csrs[KVM_CSR_GINTC]); + + /* + * We should clear linked load bit to break interrupted atomics. This + * prevents a SC on the next VCPU from succeeding by matching a LL on + * the previous VCPU. + */ + if (vcpu->kvm->created_vcpus > 1) + kvm_set_gcsr_llbctl(KVM_LLBCTL_WCLLB); + + return 0; +} + +/* Restore ASID once we are scheduled back after preemption */ +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + unsigned long flags; + + local_irq_save(flags); + vcpu->cpu = cpu; + if (vcpu->arch.last_sched_cpu != cpu) { + kvm_debug("[%d->%d]KVM VCPU[%d] switch\n", + vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); + /* + * Migrate the timer interrupt to the current CPU so that it + * always interrupts the guest and synchronously triggers a + * guest timer interrupt. + */ + kvm_migrate_count(vcpu); + } + + /* restore guest state to registers */ + _kvm_vcpu_load(vcpu, cpu); + kvm_steal_time_clear_preempted(vcpu); + local_irq_restore(flags); +} + +static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + kvm_lose_fpu(vcpu); + kvm_lose_hw_perf(vcpu); + + kvm_save_hw_gcsr(csr, KVM_CSR_CRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_PRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_EUEN); + kvm_save_hw_gcsr(csr, KVM_CSR_MISC); + kvm_save_hw_gcsr(csr, KVM_CSR_ECFG); + kvm_save_hw_gcsr(csr, KVM_CSR_ERA); + kvm_save_hw_gcsr(csr, KVM_CSR_BADV); + kvm_save_hw_gcsr(csr, KVM_CSR_BADI); + kvm_save_hw_gcsr(csr, KVM_CSR_EENTRY); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBIDX); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBEHI); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO0); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO1); + kvm_save_hw_gcsr(csr, KVM_CSR_ASID); + kvm_save_hw_gcsr(csr, KVM_CSR_PGDL); + kvm_save_hw_gcsr(csr, KVM_CSR_PGDH); + kvm_save_hw_gcsr(csr, KVM_CSR_PGD); + kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL0); + kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL1); + kvm_save_hw_gcsr(csr, KVM_CSR_STLBPGSIZE); + kvm_save_hw_gcsr(csr, KVM_CSR_RVACFG); + kvm_save_hw_gcsr(csr, KVM_CSR_CPUID); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG1); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG2); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG3); + kvm_save_hw_gcsr(csr, KVM_CSR_KS0); + kvm_save_hw_gcsr(csr, KVM_CSR_KS1); + kvm_save_hw_gcsr(csr, KVM_CSR_KS2); + kvm_save_hw_gcsr(csr, KVM_CSR_KS3); + kvm_save_hw_gcsr(csr, KVM_CSR_KS4); + kvm_save_hw_gcsr(csr, KVM_CSR_KS5); + kvm_save_hw_gcsr(csr, KVM_CSR_KS6); + kvm_save_hw_gcsr(csr, KVM_CSR_KS7); + kvm_save_hw_gcsr(csr, KVM_CSR_TMID); + kvm_save_hw_gcsr(csr, KVM_CSR_CNTC); + kvm_save_hw_gcsr(csr, KVM_CSR_LLBCTL); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRENTRY); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRBADV); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRERA); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRSAVE); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO0); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO1); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBREHI); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRPRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN0); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN1); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN2); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN3); + + /* save Root.Guestexcept in unused Guest guestexcept register */ + kvm_save_timer(vcpu); + csr->csrs[KVM_CSR_GINTC] = kvm_read_csr_gintc(); + return 0; +} + +/* ASID can change if another task is scheduled during preemption */ +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + int cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + vcpu->arch.last_sched_cpu = cpu; + vcpu->cpu = -1; + + /* save guest state in registers */ + _kvm_vcpu_put(vcpu, cpu); + kvm_steal_time_set_preempted(vcpu); + local_irq_restore(flags); +} + +static int _kvm_get_one_reg(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, s64 *v) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + int reg_idx, ret; + + if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) { + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + ret = _kvm_getcsr(vcpu, reg_idx, v, 0); + if (ret == 0) + return ret; + } + + switch (reg->id) { + case KVM_REG_LBT_SCR0: + *v = vcpu->arch.lbt.scr0; + break; + case KVM_REG_LBT_SCR1: + *v = vcpu->arch.lbt.scr1; + break; + case KVM_REG_LBT_SCR2: + *v = vcpu->arch.lbt.scr2; + break; + case KVM_REG_LBT_SCR3: + *v = vcpu->arch.lbt.scr3; + break; + case KVM_REG_LBT_FLAGS: + *v = vcpu->arch.lbt.eflags; + break; + case KVM_REG_LBT_FTOP: + *v = vcpu->arch.fpu.ftop; + break; + + case KVM_REG_LOONGARCH_COUNTER: + *v = drdtime() + vcpu->kvm->arch.stablecounter_gftoffset; + break; + default: + if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR) + return -EINVAL; + + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + if (reg_idx < CSR_ALL_SIZE) + *v = kvm_read_sw_gcsr(csr, reg_idx); + else + return -EINVAL; + } + return 0; +} + +static int _kvm_set_one_reg(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + s64 v) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + struct gfn_to_pfn_cache *cache; + int ret = 0; + unsigned long flags; + u64 val; + int reg_idx; + + val = v; + if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) { + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + ret = _kvm_setcsr(vcpu, reg_idx, &val, 0); + if (ret == 0) + return ret; + } + + switch (reg->id) { + case KVM_REG_LBT_SCR0: + vcpu->arch.lbt.scr0 = val; + break; + case KVM_REG_LBT_SCR1: + vcpu->arch.lbt.scr1 = val; + break; + case KVM_REG_LBT_SCR2: + vcpu->arch.lbt.scr2 = val; + break; + case KVM_REG_LBT_SCR3: + vcpu->arch.lbt.scr3 = val; + break; + case KVM_REG_LBT_FLAGS: + vcpu->arch.lbt.eflags = val; + break; + case KVM_REG_LBT_FTOP: + vcpu->arch.fpu.ftop = val; + break; + + case KVM_REG_LOONGARCH_COUNTER: + local_irq_save(flags); + /* + * gftoffset is relative with board, not vcpu + * only set for the first time for smp system + */ + if (!vcpu->kvm->arch.stablecounter_gftoffset) + vcpu->kvm->arch.stablecounter_gftoffset = (signed long)(v - drdtime()); + kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset); + local_irq_restore(flags); + break; + case KVM_REG_LOONGARCH_VCPU_RESET: + cache = &vcpu->arch.st.cache; + kvm_reset_timer(vcpu); + if (vcpu->vcpu_id == 0) + kvm_setup_ls3a_extirq(vcpu->kvm); + memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); + memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); + + if (vcpu->arch.st.guest_addr) { + kvm_release_pfn(cache->pfn, cache->dirty, cache); + /* disable pv timer when cpu resetting */ + vcpu->arch.st.guest_addr = 0; + } + vcpu->kvm->arch.stablecounter_gftoffset = 0; + break; + default: + if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR) + return -EINVAL; + + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + if (reg_idx < CSR_ALL_SIZE) + kvm_write_sw_gcsr(csr, reg_idx, v); + else + return -EINVAL; + } + return ret; +} + +static int _kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + int ret; + s64 v; + + ret = _kvm_get_one_reg(vcpu, reg, &v); + if (ret) + return ret; + + ret = -EINVAL; + if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { + u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; + + ret = put_user(v, uaddr64); + } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { + u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; + u32 v32 = (u32)v; + + ret = put_user(v32, uaddr32); + } + + return ret; +} + +static int _kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + s64 v; + int ret; + + ret = -EINVAL; + if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { + u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; + ret = get_user(v, uaddr64); + } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { + u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; + s32 v32; + + ret = get_user(v32, uaddr32); + v = (s64)v32; + } + + if (ret) + return -EFAULT; + + return _kvm_set_one_reg(vcpu, reg, v); +} + +static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, + struct kvm_enable_cap *cap) +{ + int r = 0; + + if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap)) + return -EINVAL; + if (cap->flags) + return -EINVAL; + if (cap->args[0]) + return -EINVAL; + + switch (cap->cap) { + case KVM_CAP_LOONGARCH_FPU: + case KVM_CAP_LOONGARCH_LSX: + break; + default: + r = -EINVAL; + break; + } + + return r; +} + +long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + + if (ioctl == KVM_INTERRUPT) { + struct kvm_loongarch_interrupt irq; + + if (copy_from_user(&irq, argp, sizeof(irq))) + return -EFAULT; + kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, + irq.irq); + + return kvm_vcpu_ioctl_interrupt(vcpu, &irq); + } + + return -ENOIOCTLCMD; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + u32 irq = irq_level->irq; + unsigned int irq_type, vcpu_idx, irq_num, ret; + int nrcpus = atomic_read(&kvm->online_vcpus); + bool level = irq_level->level; + unsigned long flags; + + irq_type = (irq >> KVM_LOONGSON_IRQ_TYPE_SHIFT) & KVM_LOONGSON_IRQ_TYPE_MASK; + vcpu_idx = (irq >> KVM_LOONGSON_IRQ_VCPU_SHIFT) & KVM_LOONGSON_IRQ_VCPU_MASK; + irq_num = (irq >> KVM_LOONGSON_IRQ_NUM_SHIFT) & KVM_LOONGSON_IRQ_NUM_MASK; + + switch (irq_type) { + case KVM_LOONGSON_IRQ_TYPE_IOAPIC: + if (!ls7a_ioapic_in_kernel(kvm)) + return -ENXIO; + + if (vcpu_idx >= nrcpus) + return -EINVAL; + + ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags); + ret = kvm_ls7a_ioapic_set_irq(kvm, irq_num, level); + ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags); + return ret; + } + kvm->stat.vm_ioctl_irq_line++; + + return -EINVAL; +} + +static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip) +{ + int r, dlen; + + r = 0; + dlen = chip->len - sizeof(struct loongarch_kvm_irqchip); + switch (chip->chip_id) { + case KVM_IRQCHIP_LS7A_IOAPIC: + if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) { + kvm_err("get ls7a state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls7a_ioapic(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_GIPI: + if (dlen != sizeof(gipiState)) { + kvm_err("get gipi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls3a_ipi(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_HT_IRQ: + case KVM_IRQCHIP_LS3A_ROUTE: + break; + case KVM_IRQCHIP_LS3A_EXTIRQ: + if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) { + kvm_err("get extioi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls3a_extirq(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_IPMASK: + break; + default: + r = -EINVAL; + break; + } + return r; +dlen_err: + r = -EINVAL; + return r; +} + +static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip) +{ + int r, dlen; + + r = 0; + dlen = chip->len - sizeof(struct loongarch_kvm_irqchip); + switch (chip->chip_id) { + case KVM_IRQCHIP_LS7A_IOAPIC: + if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) { + kvm_err("set ls7a state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls7a_ioapic(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_GIPI: + if (dlen != sizeof(gipiState)) { + kvm_err("set gipi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls3a_ipi(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_HT_IRQ: + case KVM_IRQCHIP_LS3A_ROUTE: + break; + case KVM_IRQCHIP_LS3A_EXTIRQ: + if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) { + kvm_err("set extioi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls3a_extirq(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_IPMASK: + break; + default: + r = -EINVAL; + break; + } + return r; +dlen_err: + r = -EINVAL; + return r; +} + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int _kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_csr_entry *entries, + int (*do_csr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data, int force)) +{ + int i; + + for (i = 0; i < msrs->ncsrs; ++i) + if (do_csr(vcpu, entries[i].index, &entries[i].data, 1)) + break; + + return i; +} + +static int kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_csr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data, int force)) +{ + struct kvm_msrs msrs; + struct kvm_csr_entry *entries; + int r, n; + unsigned size; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof msrs)) + goto out; + + r = -E2BIG; + if (msrs.ncsrs >= CSR_ALL_SIZE) + goto out; + + size = sizeof(struct kvm_csr_entry) * msrs.ncsrs; + entries = memdup_user(user_msrs->entries, size); + if (IS_ERR(entries)) { + r = PTR_ERR(entries); + goto out; + } + + r = n = _kvm_csr_io(vcpu, &msrs, entries, do_csr); + if (r < 0) + goto out_free; + + r = -EFAULT; + if (copy_to_user(user_msrs->entries, entries, size)) + goto out_free; + + r = n; + +out_free: + kfree(entries); +out: + return r; +} + +static int _kvm_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_set_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int _kvm_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_get_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int _kvm_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_has_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + struct kvm_device_attr attr; + long r; + + vcpu_load(vcpu); + + switch (ioctl) { + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: { + struct kvm_one_reg reg; + + r = -EFAULT; + if (copy_from_user(®, argp, sizeof(reg))) + break; + if (ioctl == KVM_SET_ONE_REG) + r = _kvm_set_reg(vcpu, ®); + else + r = _kvm_get_reg(vcpu, ®); + break; + } + case KVM_ENABLE_CAP: { + struct kvm_enable_cap cap; + + r = -EFAULT; + if (copy_from_user(&cap, argp, sizeof(cap))) + break; + r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); + break; + } + case KVM_CHECK_EXTENSION: { + unsigned int ext; + if (copy_from_user(&ext, argp, sizeof(ext))) + return -EFAULT; + switch (ext) { + case KVM_CAP_LOONGARCH_FPU: + r = !!cpu_has_fpu; + break; + case KVM_CAP_LOONGARCH_LSX: + r = !!cpu_has_lsx; + break; + default: + break; + } + break; + } + + case KVM_LOONGARCH_GET_VCPU_STATE: + { + int i; + struct kvm_loongarch_vcpu_state vcpu_state; + r = -EFAULT; + + vcpu_state.online_vcpus = vcpu->kvm->arch.online_vcpus; + vcpu_state.is_migrate = 1; + for (i = 0; i < 4; i++) + vcpu_state.core_ext_ioisr[i] = vcpu->arch.core_ext_ioisr[i]; + + vcpu_state.irq_pending = vcpu->arch.irq_pending; + vcpu_state.irq_clear = vcpu->arch.irq_clear; + + if (copy_to_user(argp, &vcpu_state, sizeof(struct kvm_loongarch_vcpu_state))) + break; + r = 0; + break; + } + + case KVM_LOONGARCH_SET_VCPU_STATE: + { + int i; + struct kvm_loongarch_vcpu_state vcpu_state; + r = -EFAULT; + + if (copy_from_user(&vcpu_state, argp, sizeof(struct kvm_loongarch_vcpu_state))) + return -EFAULT; + + vcpu->kvm->arch.online_vcpus = vcpu_state.online_vcpus; + vcpu->kvm->arch.is_migrate = vcpu_state.is_migrate; + for (i = 0; i < 4; i++) + vcpu->arch.core_ext_ioisr[i] = vcpu_state.core_ext_ioisr[i]; + + vcpu->arch.irq_pending = vcpu_state.irq_pending; + vcpu->arch.irq_clear = vcpu_state.irq_clear; + r = 0; + break; + } + case KVM_GET_MSRS: { + r = kvm_csr_io(vcpu, argp, _kvm_getcsr); + break; + } + case KVM_SET_MSRS: { + r = kvm_csr_io(vcpu, argp, _kvm_setcsr); + break; + } + case KVM_SET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_set_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_has_attr(vcpu, &attr); + break; + } + default: + r = -ENOIOCTLCMD; + } + + vcpu_put(vcpu); + return r; +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + +} + +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + /* + * FIXME: disable THP to improve vm migration success ratio, + * how to know migration failure to enable THP again + */ + memslot->arch.flags |= KVM_MEMSLOT_DISABLE_THP; + + /* Let implementation handle TLB/GVA invalidation */ + kvm_flush_remote_tlbs(kvm); +} + +long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + long r; + + switch (ioctl) { + case KVM_CREATE_IRQCHIP: + { + mutex_lock(&kvm->lock); + r = -EEXIST; + if (kvm->arch.v_ioapic) + goto create_irqchip_unlock; + + r = kvm_create_ls7a_ioapic(kvm); + if (r < 0) + goto create_irqchip_unlock; + r = kvm_create_ls3a_ipi(kvm); + if (r < 0) { + mutex_lock(&kvm->slots_lock); + kvm_destroy_ls7a_ioapic(kvm); + mutex_unlock(&kvm->slots_lock); + goto create_irqchip_unlock; + } + r = kvm_create_ls3a_ext_irq(kvm); + if (r < 0) { + mutex_lock(&kvm->slots_lock); + kvm_destroy_ls3a_ipi(kvm); + kvm_destroy_ls7a_ioapic(kvm); + mutex_unlock(&kvm->slots_lock); + } + kvm_ls7a_setup_default_irq_routing(kvm); + irqchip_debug_init(kvm); + /* Write kvm->irq_routing before kvm->arch.vpic. */ + smp_wmb(); +create_irqchip_unlock: + mutex_unlock(&kvm->lock); + break; + } + case KVM_GET_IRQCHIP: { + struct loongarch_kvm_irqchip *kchip; + struct loongarch_kvm_irqchip uchip; + if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip))) + goto out; + kchip = memdup_user(argp, uchip.len); + if (IS_ERR(kchip)) { + r = PTR_ERR(kchip); + goto out; + } + + r = -ENXIO; + if (!ls7a_ioapic_in_kernel(kvm)) + goto get_irqchip_out; + r = kvm_vm_ioctl_get_irqchip(kvm, kchip); + if (r) + goto get_irqchip_out; + if (copy_to_user(argp, kchip, kchip->len)) + goto get_irqchip_out; + r = 0; +get_irqchip_out: + kfree(kchip); + break; + } + case KVM_SET_IRQCHIP: { + struct loongarch_kvm_irqchip *kchip; + struct loongarch_kvm_irqchip uchip; + if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip))) + goto out; + + kchip = memdup_user(argp, uchip.len); + if (IS_ERR(kchip)) { + r = PTR_ERR(kchip); + goto out; + } + + r = -ENXIO; + if (!ls7a_ioapic_in_kernel(kvm)) + goto set_irqchip_out; + r = kvm_vm_ioctl_set_irqchip(kvm, kchip); + if (r) + goto set_irqchip_out; + r = 0; +set_irqchip_out: + kfree(kchip); + break; + } + case KVM_LOONGARCH_GET_IOCSR: + { + r = _kvm_get_iocsr(kvm, argp); + break; + } + case KVM_LOONGARCH_SET_IOCSR: + { + r = _kvm_set_iocsr(kvm, argp); + break; + } + case KVM_LOONGARCH_SET_CPUCFG: + { + r = 0; + if (copy_from_user(&kvm->arch.cpucfgs, argp, sizeof(struct kvm_cpucfg))) + r = -EFAULT; + break; + } + case KVM_LOONGARCH_GET_CPUCFG: + { + r = 0; + if (copy_to_user(argp, &kvm->arch.cpucfgs, sizeof(struct kvm_cpucfg))) + r = -EFAULT; + break; + } + default: + r = -ENOIOCTLCMD; + } +out: + + return r; +} + +int kvm_arch_init(void *opaque) +{ + struct kvm_context *context; + unsigned long vpid_mask; + int cpu; + + vmcs = alloc_percpu(struct kvm_context); + if (!vmcs) { + printk(KERN_ERR "kvm: failed to allocate percpu kvm_context\n"); + return -ENOMEM; + } + + vpid_mask = kvm_read_csr_gstat(); + vpid_mask = (vpid_mask & KVM_GSTAT_GIDBIT) >> KVM_GSTAT_GIDBIT_SHIFT; + if (vpid_mask) + vpid_mask = GENMASK(vpid_mask - 1, 0); + + for_each_possible_cpu(cpu) { + context = per_cpu_ptr(vmcs, cpu); + context->gid_mask = vpid_mask; + context->gid_ver_mask = ~context->gid_mask; + context->gid_fisrt_ver = context->gid_mask + 1; + context->vpid_cache = context->gid_mask + 1; + context->last_vcpu = NULL; + } + + _kvm_init_fault(); + return 0; +} + +void kvm_arch_exit(void) +{ + free_percpu(vmcs); +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + int i = 0; + + /* no need vcpu_load and vcpu_put */ + fpu->fcsr = vcpu->arch.fpu.fcsr; + fpu->fcc = vcpu->arch.fpu.fcc; + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + int i = 0; + + /* no need vcpu_load and vcpu_put */ + vcpu->arch.fpu.fcsr = fpu->fcsr; + vcpu->arch.fpu.fcc = fpu->fcc; + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); + + return 0; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_ONE_REG: + case KVM_CAP_ENABLE_CAP: + case KVM_CAP_READONLY_MEM: + case KVM_CAP_SYNC_MMU: +#ifdef CONFIG_HAVE_LS_KVM_MSI + case KVM_CAP_SIGNAL_MSI: +#endif + case KVM_CAP_IMMEDIATE_EXIT: + r = 1; + break; + case KVM_CAP_NR_VCPUS: + r = num_online_cpus(); + break; + case KVM_CAP_MAX_VCPUS: + r = KVM_MAX_VCPUS; + break; + case KVM_CAP_MAX_VCPU_ID: + r = KVM_MAX_VCPU_ID; + break; + case KVM_CAP_LOONGARCH_FPU: + /* We don't handle systems with inconsistent cpu_has_fpu */ + r = !!cpu_has_fpu; + break; + case KVM_CAP_LOONGARCH_LSX: + /* + * We don't support LSX vector partitioning yet: + * 1) It would require explicit support which can't be tested + * yet due to lack of support in current hardware. + * 2) It extends the state that would need to be saved/restored + * by e.g. QEMU for migration. + * + * When vector partitioning hardware becomes available, support + * could be added by requiring a flag when enabling + * KVM_CAP_LOONGARCH_LSX capability to indicate that userland knows + * to save/restore the appropriate extra state. + */ + r = cpu_has_lsx; + break; + case KVM_CAP_IRQCHIP: + case KVM_CAP_IOEVENTFD: + /* we wouldn't be here unless cpu_has_lvz */ + r = 1; + break; + case KVM_CAP_LOONGARCH_VZ: + /* get user defined kvm version */ + r = KVM_LOONGARCH_VERSION; + break; + default: + r = 0; + break; + } + return r; +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return _kvm_pending_timer(vcpu) || + kvm_read_hw_gcsr(KVM_CSR_ESTAT) & (1 << INT_TI); +} + +int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) +{ + int i; + struct loongarch_csrs *csr; + + if (!vcpu) + return -1; + + kvm_debug("VCPU Register Dump:\n"); + kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc); + kvm_debug("\texceptions: %08lx\n", vcpu->arch.irq_pending); + + for (i = 0; i < 32; i += 4) { + kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i, + vcpu->arch.gprs[i], + vcpu->arch.gprs[i + 1], + vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); + } + + csr = vcpu->arch.csr; + kvm_debug("\tCRMOD: 0x%08lx, exst: 0x%08lx\n", + kvm_read_hw_gcsr(KVM_CSR_CRMD), + kvm_read_hw_gcsr(KVM_CSR_ESTAT)); + + kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(KVM_CSR_ERA)); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + int i; + + vcpu_load(vcpu); + + for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) + vcpu->arch.gprs[i] = regs->gpr[i]; + vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ + vcpu->arch.pc = regs->pc; + + vcpu_put(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + int i; + + vcpu_load(vcpu); + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) + regs->gpr[i] = vcpu->arch.gprs[i]; + + regs->pc = vcpu->arch.pc; + + vcpu_put(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return 0; +} + +#ifdef CONFIG_CPU_HAS_LBT +/* Enable FPU for guest and restore context */ +void kvm_own_lbt(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + if (!(vcpu->arch.aux_inuse & KVM_LARCH_LBT)) { + kvm_set_csr_euen(KVM_EUEN_LBTEN); + + /* If guest lbt state not active, restore it now */ + kvm_restore_lbt(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LBT; + } + preempt_enable(); +} + +static void kvm_enable_lbt_fpu(struct kvm_vcpu *vcpu, unsigned long fcsr) +{ + /* + * if TM is enabled, top register save/restore will + * cause lbt exception, here enable lbt in advanced + */ + if (fcsr & FPU_CSR_TM) + kvm_own_lbt(vcpu); +} + +static void kvm_lose_lbt(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) { + kvm_save_lbt(vcpu); + kvm_clear_csr_euen(KVM_EUEN_LBTEN); + vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT; + } + preempt_enable(); +} + +#else +void kvm_own_lbt(struct kvm_vcpu *vcpu) { } +static void kvm_enable_lbt_fpu(struct kvm_vcpu *vcpu, unsigned long fcsr) { } +static void kvm_lose_lbt(struct kvm_vcpu *vcpu) { } +#endif + +/* Enable FPU for guest and restore context */ +void kvm_own_fpu(struct kvm_vcpu *vcpu) +{ + + preempt_disable(); + + /* + * Enable FPU for guest + * We set FR and FRE according to guest context + */ + kvm_enable_lbt_fpu(vcpu, vcpu->arch.fpu.fcsr); + kvm_set_csr_euen(KVM_EUEN_FPEN); + + /* If guest FPU state not active, restore it now */ + kvm_restore_fpu(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_FPU; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); + + preempt_enable(); +} + +#ifdef CONFIG_CPU_HAS_LSX +/* Enable LSX for guest and restore context */ +void kvm_own_lsx(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + + /* Enable LSX for guest */ + kvm_enable_lbt_fpu(vcpu, vcpu->arch.fpu.fcsr); + kvm_set_csr_euen(KVM_EUEN_LSXEN | KVM_EUEN_FPEN); + switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + case KVM_LARCH_FPU: + /* + * Guest FPU state already loaded, + * only restore upper LSX state + */ + kvm_restore_lsx_upper(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_LSX); + break; + default: + /* Neither FP or LSX already active, + * restore full LSX state + */ + kvm_restore_lsx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_FPU_LSX); + break; + } + + vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU; + preempt_enable(); +} +#endif + +#ifdef CONFIG_CPU_HAS_LASX +/* Enable LASX for guest and restore context */ +void kvm_own_lasx(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + + kvm_enable_lbt_fpu(vcpu, vcpu->arch.fpu.fcsr); + kvm_set_csr_euen(KVM_EUEN_FPEN | KVM_EUEN_LSXEN | KVM_EUEN_LASXEN); + switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) { + case KVM_LARCH_LSX | KVM_LARCH_FPU: + case KVM_LARCH_LSX: + /* + * Guest LSX state already loaded, only restore upper LASX state + */ + kvm_restore_lasx_upper(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); + break; + case KVM_LARCH_FPU: + /* + * Guest FP state already loaded, only restore 64~256 LASX state + */ + kvm_restore_lsx_upper(vcpu); + kvm_restore_lasx_upper(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); + break; + default: + /* Neither FP or LSX already active, restore full LASX state */ + kvm_restore_lasx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_FPU_LSX_LASX); + break; + } + + vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU; + preempt_enable(); +} +#endif + +/* Save and disable FPU & LSX & LASX */ +void kvm_lose_fpu(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_CPU_HAS_LBT + unsigned long fcsr; +#endif + + preempt_disable(); +#ifdef CONFIG_CPU_HAS_LBT + if (vcpu->arch.aux_inuse & KVM_LARCH_FP_ALL) { + if (!(vcpu->arch.aux_inuse & KVM_LARCH_LBT)) { + fcsr = read_fcsr(LOONGARCH_FCSR0); + kvm_enable_lbt_fpu(vcpu, fcsr); + } + } +#endif + if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) { + kvm_save_lasx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX_LASX); + /* Disable LASX & MAS & FPU */ + kvm_clear_csr_euen(KVM_EUEN_FPEN | KVM_EUEN_LSXEN | KVM_EUEN_LASXEN); + } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + kvm_save_lsx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX); + /* Disable LSX & FPU */ + kvm_clear_csr_euen(KVM_EUEN_FPEN | KVM_EUEN_LSXEN); + } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + kvm_save_fpu(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); + /* Disable FPU */ + kvm_clear_csr_euen(KVM_EUEN_FPEN); + } + vcpu->arch.aux_inuse &= ~KVM_LARCH_FP_ALL; + + kvm_lose_lbt(vcpu); + preempt_enable(); +} + +void kvm_lose_hw_perf(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) { + struct loongarch_csrs *csr = vcpu->arch.csr; + /* save guest pmu csr */ + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL0); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR0); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL1); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR1); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL2); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR2); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL3); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR3); + if (((kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL0) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL1) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL2) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL3)) + & KVM_PMU_PLV_ENABLE) == 0) + vcpu->arch.aux_inuse &= ~KVM_LARCH_PERF; + /* config host pmu csr */ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() & ~KVM_GCFG_GPERF); + /* TODO: pmu csr used by host and guest at the same time */ + kvm_write_csr_perfctrl0(0); + kvm_write_csr_perfcntr0(0); + kvm_write_csr_perfctrl1(0); + kvm_write_csr_perfcntr1(0); + kvm_write_csr_perfctrl2(0); + kvm_write_csr_perfcntr2(0); + kvm_write_csr_perfctrl3(0); + kvm_write_csr_perfcntr3(0); + } +} + +void kvm_restore_hw_perf(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) { + struct loongarch_csrs *csr = vcpu->arch.csr; + /* enable guest pmu */ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL1); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR1); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL2); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR2); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL3); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR3); + } +} + +static int __init kvm_loongarch_init(void) +{ + int ret; + + if (!cpu_has_lvz) + return 0; + + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) + return ret; + + return 0; +} + +static void __exit kvm_loongarch_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_loongarch_init); +module_exit(kvm_loongarch_exit); + +static const struct cpu_feature loongarch_kvm_feature[] = { + { .feature = cpu_feature(LOONGARCH_LVZ) }, + {}, +}; +MODULE_DEVICE_TABLE(cpu, loongarch_kvm_feature); + +EXPORT_TRACEPOINT_SYMBOL(kvm_exit); diff --git a/arch/loongarch/kvm/ls_irq.h b/arch/loongarch/kvm/ls_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..ef0af6032a6996acf62b3f5db81ed79212904057 --- /dev/null +++ b/arch/loongarch/kvm/ls_irq.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGHARCH_KVM_IRQ_H__ +#define __LOONGHARCH_KVM_IRQ_H__ + +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return kvm->arch.v_ioapic ? 1 : 0; +} + +int kvm_ls7a_setup_default_irq_routing(struct kvm *kvm); +#endif diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c new file mode 100644 index 0000000000000000000000000000000000000000..2fdaa05f15a6b22a25a91b71003a9d2298ef8711 --- /dev/null +++ b/arch/loongarch/kvm/mmu.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvm_compat.h" + +/* + * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels + * for which pages need to be cached. + */ +#if defined(__PAGETABLE_PMD_FOLDED) +#define KVM_MMU_CACHE_MIN_PAGES 1 +#else +#define KVM_MMU_CACHE_MIN_PAGES 2 +#endif + +static inline int kvm_pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; } +static inline pte_t kvm_pte_mksmall(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_HUGE; + return pte; +} +static inline void kvm_set_pte(pte_t *ptep, pte_t val) +{ + WRITE_ONCE(*ptep, val); +} + +static int kvm_tlb_flush_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +{ + preempt_disable(); + gpa &= (PAGE_MASK << 1); + invtlb(INVTLB_GID_ADDR, kvm_read_csr_gstat() & KVM_GSTAT_GID, gpa); + preempt_enable(); + return 0; +} + +static inline pmd_t kvm_pmd_mkhuge(pmd_t pmd) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkhuge(pmd); +#elif defined(CONFIG_HUGETLB_PAGE) + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkhuge(entry); + pmd_val(pmd) = pte_val(entry); +#endif + return pmd; +} + +static inline pmd_t kvm_pmd_mkclean(pmd_t pmd) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkclean(pmd); +#elif defined(CONFIG_HUGETLB_PAGE) + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkclean(entry); + pmd_val(pmd) = pte_val(entry); +#endif + return pmd; +} + +static inline pmd_t kvm_pmd_mkold(pmd_t pmd) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkold(pmd); +#elif defined(CONFIG_HUGETLB_PAGE) + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkold(entry); + pmd_val(pmd) = pte_val(entry); +#endif + return pmd; +} + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); +} + +/** + * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory. + * + * Allocate a blank KVM GPA page directory (PGD) for representing guest physical + * to host physical page mappings. + * + * Returns: Pointer to new KVM GPA page directory. + * NULL on allocation failure. + */ +pgd_t *kvm_pgd_alloc(void) +{ + pgd_t *ret; + struct page *page; + + page = alloc_pages(GFP_KERNEL, 0); + if (!page) + return NULL; + ret = (pgd_t *) page_address(page); + if (ret) + pgd_init((unsigned long)ret); + + return ret; +} + +/** + * kvm_walk_pgd() - Walk page table with optional allocation. + * @pgd: Page directory pointer. + * @addr: Address to index page table using. + * @cache: MMU page cache to allocate new page tables from, or NULL. + * + * Walk the page tables pointed to by @pgd to find the PTE corresponding to the + * address @addr. If page tables don't exist for @addr, they will be created + * from the MMU cache if @cache is not NULL. + * + * Returns: Pointer to pte_t corresponding to @addr. + * NULL if a page table doesn't exist for @addr and !@cache. + * NULL if a page table allocation failed. + */ +static pte_t *kvm_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, + unsigned long addr) +{ + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd += pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used yet */ + BUG(); + return NULL; + } + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd_t *new_pmd; + + if (!cache) + return NULL; + new_pmd = kvm_mmu_memory_cache_alloc(cache); + pmd_init((unsigned long)new_pmd, + (unsigned long)invalid_pte_table); + pud_populate(NULL, pud, new_pmd); + } + pmd = pmd_offset(pud, addr); +#ifdef CONFIG_HUGETLB_PAGE + if (pmd_huge(*pmd)) { + return (pte_t *)pmd; + } +#endif + if (pmd_none(*pmd)) { + pte_t *new_pte; + + if (!cache) + return NULL; + new_pte = kvm_mmu_memory_cache_alloc(cache); + clear_page(new_pte); + pmd_populate_kernel(NULL, pmd, new_pte); + } + return pte_offset_kernel(pmd, addr); +} + +/* Caller must hold kvm->mm_lock */ +static pte_t *kvm_pte_for_gpa(struct kvm *kvm, + struct kvm_mmu_memory_cache *cache, + unsigned long addr) +{ + return kvm_walk_pgd(kvm->arch.gpa_mm.pgd, cache, addr); +} + +/* + * kvm_flush_gpa_{pte,pmd,pud,pgd,pt}. + * Flush a range of guest physical address space from the VM's GPA page tables. + */ + +static bool kvm_flush_gpa_pte(pte_t *pte, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + int i_min = pte_index(start_gpa); + int i_max = pte_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); + int i; + + for (i = i_min; i <= i_max; ++i) { + if (!pte_present(pte[i])) + continue; + + set_pte(pte + i, __pte(0)); + if (data) + *data = *data + 1; + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + pte_t *pte; + unsigned long end = ~0ul; + int i_min = pmd_index(start_gpa); + int i_max = pmd_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (pmd_huge(pmd[i]) && pmd_present(pmd[i])) { + pmd_clear(pmd + i); + if (data) + *data += PTRS_PER_PMD; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pte(pte, start_gpa, end, data)) { + pmd_clear(pmd + i); + pte_free_kernel(NULL, pte); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pud(pud_t *pud, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + pmd_t *pmd; + unsigned long end = ~0ul; + int i_min = pud_index(start_gpa); + int i_max = pud_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pmd(pmd, start_gpa, end, data)) { + pud_clear(pud + i); + pmd_free(NULL, pmd); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + p4d_t *p4d; + pud_t *pud; + unsigned long end = ~0ul; + int i_min = pgd_index(start_gpa); + int i_max = pgd_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd, 0); + pud = pud_offset(p4d + i, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pud(pud, start_gpa, end, data)) { + pgd_clear(pgd + i); + pud_free(NULL, pud); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +/** + * kvm_flush_gpa_pt() - Flush a range of guest physical addresses. + * @kvm: KVM pointer. + * @start_gfn: Guest frame number of first page in GPA range to flush. + * @end_gfn: Guest frame number of last page in GPA range to flush. + * + * Flushes a range of GPA mappings from the GPA page tables. + * + * The caller must hold the @kvm->mmu_lock spinlock. + * + * Returns: Whether its safe to remove the top level page directory because + * all lower levels have been removed. + */ +static bool kvm_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, void *data) +{ + return kvm_flush_gpa_pgd(kvm->arch.gpa_mm.pgd, + start_gfn << PAGE_SHIFT, + end_gfn << PAGE_SHIFT, (unsigned long *)data); +} + +/* + * kvm_mkclean_gpa_pt. + * Mark a range of guest physical address space clean (writes fault) in the VM's + * GPA page table to allow dirty page tracking. + */ + +static int kvm_mkclean_pte(pte_t *pte, unsigned long start, unsigned long end) +{ + int ret = 0; + int i_min = pte_index(start); + int i_max = pte_index(end); + int i; + pte_t val; + + for (i = i_min; i <= i_max; ++i) { + val = pte[i]; + if (pte_present(val) && pte_dirty(val)) { + set_pte(pte + i, pte_mkclean(val)); + ret = 1; + } + } + return ret; +} + +static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end) +{ + int ret = 0; + pte_t *pte; + unsigned long cur_end = ~0ul; + int i_min = pmd_index(start); + int i_max = pmd_index(end); + int i; + pmd_t old, new; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (pmd_huge(pmd[i])) { + old = pmd[i]; + new = kvm_pmd_mkclean(old); + if (pmd_val(new) == pmd_val(old)) + continue; + set_pmd(pmd + i, new); + ret = 1; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pte(pte, start, cur_end); + } + + return ret; +} + +static int kvm_mkclean_pud(pud_t *pud, unsigned long start, unsigned long end) +{ + int ret = 0; + pmd_t *pmd; + unsigned long cur_end = ~0ul; + int i_min = pud_index(start); + int i_max = pud_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pmd(pmd, start, cur_end); + } + return ret; +} + +static int kvm_mkclean_pgd(pgd_t *pgd, unsigned long start, unsigned long end) +{ + int ret = 0; + p4d_t *p4d; \ + pud_t *pud; + unsigned long cur_end = ~0ul; + int i_min = pgd_index(start); + int i_max = pgd_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd, 0); + pud = pud_offset(p4d + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pud(pud, start, cur_end); + } + return ret; +} + +/** + * kvm_mkclean_gpa_pt() - Make a range of guest physical addresses clean. + * @kvm: KVM pointer. + * @start_gfn: Guest frame number of first page in GPA range to flush. + * @end_gfn: Guest frame number of last page in GPA range to flush. + * + * Make a range of GPA mappings clean so that guest writes will fault and + * trigger dirty page logging. + * + * The caller must hold the @kvm->mmu_lock spinlock. + * + * Returns: Whether any GPA mappings were modified, which would require + * derived mappings (GVA page tables & TLB enties) to be + * invalidated. + */ +static int kvm_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn) +{ + return kvm_mkclean_pgd(kvm->arch.gpa_mm.pgd, start_gfn << PAGE_SHIFT, + end_gfn << PAGE_SHIFT); +} + +/** + * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory + * slot to be write protected + * + * Walks bits set in mask write protects the associated pte's. Caller must + * acquire @kvm->mmu_lock. + */ +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + gfn_t base_gfn = slot->base_gfn + gfn_offset; + gfn_t start = base_gfn + __ffs(mask); + gfn_t end = base_gfn + __fls(mask); + + kvm_mkclean_gpa_pt(kvm, start, end); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + int needs_flush; + + kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n", + __func__, kvm, mem->slot, mem->guest_phys_addr, + mem->memory_size, mem->userspace_addr); + + /* + * If dirty page logging is enabled, write protect all pages in the slot + * ready for dirty logging. + * + * There is no need to do this in any of the following cases: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + */ + if (change == KVM_MR_FLAGS_ONLY && + (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { + spin_lock(&kvm->mmu_lock); + /* Write protect GPA page table entries */ + needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn, + new->base_gfn + new->npages - 1); + /* Let implementation do the rest */ + if (needs_flush) + kvm_flush_remote_tlbs(kvm); + spin_unlock(&kvm->mmu_lock); + } +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ + /* Flush whole GPA */ + kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL); + + /* Flush vpid for each VCPU individually */ + kvm_flush_remote_tlbs(kvm); +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + unsigned long npages; + + /* + * The slot has been made invalid (ready for moving or deletion), so we + * need to ensure that it can no longer be accessed by any guest VCPUs. + */ + + npages = 0; + spin_lock(&kvm->mmu_lock); + /* Flush slot from GPA */ + kvm_flush_gpa_pt(kvm, slot->base_gfn, + slot->base_gfn + slot->npages - 1, &npages); + /* Let implementation do the rest */ + if (npages) + kvm_flush_remote_tlbs(kvm); + spin_unlock(&kvm->mmu_lock); +} + +void _kvm_destroy_mm(struct kvm *kvm) +{ + /* It should always be safe to remove after flushing the whole range */ + WARN_ON(!kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL)); + pgd_free(NULL, kvm->arch.gpa_mm.pgd); + kvm->arch.gpa_mm.pgd = NULL; +} + +/* + * Mark a range of guest physical address space old (all accesses fault) in the + * VM's GPA page table to allow detection of commonly used pages. + */ + +static int kvm_mkold_pte(pte_t *pte, unsigned long start, + unsigned long end) +{ + int ret = 0; + int i_min = pte_index(start); + int i_max = pte_index(end); + int i; + pte_t old, new; + + for (i = i_min; i <= i_max; ++i) { + if (!pte_present(pte[i])) + continue; + + old = pte[i]; + new = pte_mkold(old); + if (pte_val(new) == pte_val(old)) + continue; + set_pte(pte + i, new); + ret = 1; + } + + return ret; +} + +static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end) +{ + int ret = 0; + pte_t *pte; + unsigned long cur_end = ~0ul; + int i_min = pmd_index(start); + int i_max = pmd_index(end); + int i; + pmd_t old, new; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (pmd_huge(pmd[i])) { + old = pmd[i]; + new = kvm_pmd_mkold(old); + if (pmd_val(new) == pmd_val(old)) + continue; + set_pmd(pmd + i, new); + ret = 1; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pte(pte, start, cur_end); + } + + return ret; +} + +static int kvm_mkold_pud(pud_t *pud, unsigned long start, unsigned long end) +{ + int ret = 0; + pmd_t *pmd; + unsigned long cur_end = ~0ul; + int i_min = pud_index(start); + int i_max = pud_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pmd(pmd, start, cur_end); + } + + return ret; +} + +static int kvm_mkold_pgd(pgd_t *pgd, unsigned long start, unsigned long end) +{ + int ret = 0; + p4d_t *p4d; + pud_t *pud; + unsigned long cur_end = ~0ul; + int i_min = pgd_index(start); + int i_max = pgd_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd, 0); + pud = pud_offset(p4d + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pud(pud, start, cur_end); + } + + return ret; +} + +static int handle_hva_to_gpa(struct kvm *kvm, + unsigned long start, + unsigned long end, + int (*handler)(struct kvm *kvm, gfn_t gfn, + gpa_t gfn_end, + struct kvm_memory_slot *memslot, + void *data), + void *data) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int ret = 0; + + slots = kvm_memslots(kvm); + + /* we only care about the pages that the guest sees */ + kvm_for_each_memslot(memslot, slots) { + unsigned long hva_start, hva_end; + gfn_t gfn, gfn_end; + + hva_start = max(start, memslot->userspace_addr); + hva_end = min(end, memslot->userspace_addr + + (memslot->npages << PAGE_SHIFT)); + if (hva_start >= hva_end) + continue; + + /* + * {gfn(page) | page intersects with [hva_start, hva_end)} = + * {gfn_start, gfn_start+1, ..., gfn_end-1}. + */ + gfn = hva_to_gfn_memslot(hva_start, memslot); + gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); + ret |= handler(kvm, gfn, gfn_end, memslot, data); + } + + return ret; +} + + +static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + unsigned long npages; + + npages = 0; + kvm_flush_gpa_pt(kvm, gfn, gfn_end - 1, &npages); + *(unsigned long *)data = *(unsigned long *)data + npages; + + return npages > 0; +} + +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable) +{ + unsigned long npages; + + npages = 0; + return handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, &npages); +} + +static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + gpa_t gpa = gfn << PAGE_SHIFT; + pte_t hva_pte = *(pte_t *)data; + pte_t *gpa_pte = kvm_pte_for_gpa(kvm, NULL, gpa); + pte_t old_pte; + + if (!gpa_pte) + return 0; + + /* Mapping may need adjusting depending on memslot flags */ + old_pte = *gpa_pte; + if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte)) + hva_pte = pte_mkclean(hva_pte); + else if (memslot->flags & KVM_MEM_READONLY) + hva_pte = pte_wrprotect(hva_pte); + + set_pte(gpa_pte, hva_pte); + + /* Replacing an absent or old page doesn't need flushes */ + if (!pte_present(old_pte) || !pte_young(old_pte)) + return 0; + + /* Pages swapped, aged, moved, or cleaned require flushes */ + return !pte_present(hva_pte) || + !pte_young(hva_pte) || + pte_pfn(old_pte) != pte_pfn(hva_pte) || + (pte_dirty(old_pte) && !pte_dirty(hva_pte)); +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + unsigned long end = hva + PAGE_SIZE; + int ret; + + ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte); + if (ret) + /* Flush vpid for each VCPU individually */ + kvm_flush_remote_tlbs(kvm); + return 0; +} + +static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + return kvm_mkold_pgd(kvm->arch.gpa_mm.pgd, gfn << PAGE_SHIFT, + gfn_end << PAGE_SHIFT); +} + +static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + gpa_t gpa = gfn << PAGE_SHIFT; + pte_t *gpa_pte = kvm_pte_for_gpa(kvm, NULL, gpa); + + if (!gpa_pte) + return 0; + return pte_young(*gpa_pte); +} + +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) +{ + return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); +} + +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) +{ + return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); +} + +static pud_t *kvm_get_pud(struct kvm *kvm, + struct kvm_mmu_memory_cache *cache, phys_addr_t addr) +{ + pgd_t *pgd; + + pgd = kvm->arch.gpa_mm.pgd + pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used yet */ + BUG(); + return NULL; + } + + return pud_offset(p4d_offset(pgd, addr), addr); +} + +static pmd_t *kvm_get_pmd(struct kvm *kvm, + struct kvm_mmu_memory_cache *cache, phys_addr_t addr) +{ + pud_t *pud; + pmd_t *pmd; + + pud = kvm_get_pud(kvm, cache, addr); + if (!pud || pud_huge(*pud)) + return NULL; + + if (pud_none(*pud)) { + if (!cache) + return NULL; + pmd = kvm_mmu_memory_cache_alloc(cache); + pmd_init((unsigned long)pmd, + (unsigned long)invalid_pte_table); + pud_populate(NULL, pud, pmd); + } + + return pmd_offset(pud, addr); +} + +static int kvm_set_pmd_huge(struct kvm_vcpu *vcpu, struct kvm_mmu_memory_cache + *cache, phys_addr_t addr, const pmd_t *new_pmd) +{ + pmd_t *pmd, old_pmd; + +retry: + pmd = kvm_get_pmd(vcpu->kvm, cache, addr); + VM_BUG_ON(!pmd); + + old_pmd = *pmd; + /* + * Multiple vcpus faulting on the same PMD entry, can + * lead to them sequentially updating the PMD with the + * same value. Following the break-before-make + * (pmd_clear() followed by tlb_flush()) process can + * hinder forward progress due to refaults generated + * on missing translations. + * + * Skip updating the page table if the entry is + * unchanged. + */ + if (pmd_val(old_pmd) == pmd_val(*new_pmd)) + return 0; + + if (pmd_present(old_pmd)) { + /* + * If we already have PTE level mapping for this block, + * we must unmap it to avoid inconsistent TLB state and + * leaking the table page. We could end up in this situation + * if the memory slot was marked for dirty logging and was + * reverted, leaving PTE level mappings for the pages accessed + * during the period. So, unmap the PTE level mapping for this + * block and retry, as we could have released the upper level + * table in the process. + * + * Normal THP split/merge follows mmu_notifier callbacks and do + * get handled accordingly. + */ + if (!pmd_huge(old_pmd)) { + ++vcpu->stat.huge_merge_exits; + kvm_flush_gpa_pt(vcpu->kvm, + (addr & PMD_MASK) >> PAGE_SHIFT, + ((addr & PMD_MASK) + PMD_SIZE - 1) >> PAGE_SHIFT, NULL); + goto retry; + } + /* + * Mapping in huge pages should only happen through a + * fault. If a page is merged into a transparent huge + * page, the individual subpages of that huge page + * should be unmapped through MMU notifiers before we + * get here. + * + * Merging of CompoundPages is not supported; they + * should become splitting first, unmapped, merged, + * and mapped back in on-demand. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + WARN_ON_ONCE(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd)); +#endif + pmd_clear(pmd); + } + + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + set_pmd(pmd, *new_pmd); + return 0; +} + +/* + * Adjust pfn start boundary if support for transparent hugepage + */ +static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, unsigned long *gpap) +{ + kvm_pfn_t pfn = *pfnp; + gfn_t gfn = *gpap >> PAGE_SHIFT; + struct page *page = pfn_to_page(pfn); + + /* + * PageTransCompoundMap() returns true for THP and + * hugetlbfs. Make sure the adjustment is done only for THP + * pages. + */ + if ((!PageHuge(page)) && PageTransCompound(page) && + (atomic_read(&page->_mapcount) < 0)) { + unsigned long mask; + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + mask = PTRS_PER_PMD - 1; + VM_BUG_ON((gfn & mask) != (pfn & mask)); + if (pfn & mask) { + *gpap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~mask; + kvm_get_pfn(pfn); + *pfnp = pfn; + } + + return true; + } + + return false; +} + +static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, bool write) +{ + gpa_t gpa_start; + hva_t uaddr_start, uaddr_end; + unsigned long map_size; + size_t size; + + map_size = PMD_SIZE; + /* Disable dirty logging on HugePages */ + if ((memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) && write) + return false; + + size = memslot->npages * PAGE_SIZE; + gpa_start = memslot->base_gfn << PAGE_SHIFT; + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD/PUD for userspace and GPA cannot be mapped with stage-2 + * PMD/PUD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SIZE: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva & ~(map_size - 1)) >= uaddr_start && + (hva & ~(map_size - 1)) + map_size <= uaddr_end; +} + +/** + * kvm_map_page_fast() - Fast path GPA fault handler. + * @vcpu: VCPU pointer. + * @gpa: Guest physical address of fault. + * @write: Whether the fault was due to a write. + * + * Perform fast path GPA fault handling, doing all that can be done without + * calling into KVM. This handles marking old pages young (for idle page + * tracking), and dirtying of clean pages (for dirty page logging). + * + * Returns: 0 on success, in which case we can update derived mappings and + * resume guest execution. + * -EFAULT on failure due to absent GPA mapping or write to + * read-only page, in which case KVM must be consulted. + */ +static int kvm_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa, + bool write) +{ + struct kvm *kvm = vcpu->kvm; + gfn_t gfn = gpa >> PAGE_SHIFT; + pte_t *ptep; + kvm_pfn_t pfn = 0; /* silence bogus GCC warning */ + bool pfn_valid = false; + int ret = 0; + struct kvm_memory_slot *slot; + + spin_lock(&kvm->mmu_lock); + + /* Fast path - just check GPA page table for an existing entry */ + ptep = kvm_pte_for_gpa(kvm, NULL, gpa); + if (!ptep || !pte_present(*ptep)) { + ret = -EFAULT; + goto out; + } + + /* Track access to pages marked old */ + if (!pte_young(*ptep)) { + set_pte(ptep, pte_mkyoung(*ptep)); + pfn = pte_pfn(*ptep); + pfn_valid = true; + /* call kvm_set_pfn_accessed() after unlock */ + } + if (write && !pte_dirty(*ptep)) { + if (!pte_write(*ptep)) { + ret = -EFAULT; + goto out; + } + + if (kvm_pte_huge(*ptep)) { + /* + * Do not set write permission when dirty logging is + * enabled for HugePages + */ + slot = gfn_to_memslot(kvm, gfn); + if (slot->flags & KVM_MEM_LOG_DIRTY_PAGES) { + ret = -EFAULT; + goto out; + } + } + + /* Track dirtying of writeable pages */ + set_pte(ptep, pte_mkdirty(*ptep)); + pfn = pte_pfn(*ptep); + if (pmd_huge(*((pmd_t *)ptep))) { + int i; + gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT; + + for (i = 0; i < PTRS_PER_PTE; i++) + mark_page_dirty(kvm, base_gfn + i); + } else + mark_page_dirty(kvm, gfn); + kvm_set_pfn_dirty(pfn); + } + +out: + spin_unlock(&kvm->mmu_lock); + if (pfn_valid) + kvm_set_pfn_accessed(pfn); + return ret; +} + +/* + * Split huge page + */ +static pte_t *kvm_split_huge(struct kvm_vcpu *vcpu, pte_t *ptep, gfn_t gfn, + struct vm_area_struct *vma, unsigned long hva) +{ + int i; + pte_t val, *child; + struct kvm_mmu_memory_cache *memcache; + + memcache = &vcpu->arch.mmu_page_cache; + child = kvm_mmu_memory_cache_alloc(memcache); + val = kvm_pte_mksmall(*ptep); + for (i = 0; i < PTRS_PER_PTE; i++) { + kvm_set_pte(child + i, val); + pte_val(val) += PAGE_SIZE; + } + + /* The later kvm_flush_tlb_gpa() will flush hugepage tlb */ + pte_val(val) = (unsigned long)child; + kvm_set_pte(ptep, val); + return child + (gfn & (PTRS_PER_PTE - 1)); +} + +/** + * kvm_map_page() - Map a guest physical page. + * @vcpu: VCPU pointer. + * @gpa: Guest physical address of fault. + * @write: Whether the fault was due to a write. + * + * Handle GPA faults by creating a new GPA mapping (or updating an existing + * one). + * + * This takes care of marking pages young or dirty (idle/dirty page tracking), + * asking KVM for the corresponding PFN, and creating a mapping in the GPA page + * tables. Derived mappings (GVA page tables and TLBs) must be handled by the + * caller. + * + * Returns: 0 on success, in which case the caller may use the @out_entry + * and @out_buddy PTEs to update derived mappings and resume guest + * execution. + * -EFAULT if there is no memory region at @gpa or a write was + * attempted to a read-only memory region. This is usually handled + * as an MMIO access. + */ +static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, + bool write) +{ + bool writeable; + bool force_pte = false; + int i, srcu_idx, err = 0, retry_no = 0; + unsigned long hva; + unsigned long mmu_seq; + unsigned long prot_bits; + unsigned long vma_pagesize; + pte_t *ptep; + kvm_pfn_t pfn; + gfn_t gfn = gpa >> PAGE_SHIFT; + struct vm_area_struct *vma; + struct kvm *kvm = vcpu->kvm; + struct kvm_memory_slot *memslot; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + + /* Try the fast path to handle old / clean pages */ + srcu_idx = srcu_read_lock(&kvm->srcu); + err = kvm_map_page_fast(vcpu, gpa, write); + if (!err) + goto out; + + memslot = gfn_to_memslot(kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writeable); + if (kvm_is_error_hva(hva) || (write && !writeable)) + goto out; + + /* Let's check if we will get back a huge page backed by hugetlbfs */ + mmap_read_lock(current->mm); + vma = find_vma_intersection(current->mm, hva, hva + 1); + if (unlikely(!vma)) { + kvm_err("Failed to find VMA for hva 0x%lx\n", hva); + mmap_read_unlock(current->mm); + err = -EFAULT; + goto out; + } + + vma_pagesize = vma_kernel_pagesize(vma); + if ((vma_pagesize == PMD_SIZE) && + !fault_supports_huge_mapping(memslot, hva, write)) { + force_pte = true; + vma_pagesize = PAGE_SIZE; + ++vcpu->stat.huge_dec_exits; + } + + /* PMD is not folded, adjust gfn to new boundary */ + if (vma_pagesize == PMD_SIZE) + gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + + mmap_read_unlock(current->mm); + + /* We need a minimum of cached pages ready for page table creation */ + err = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES); + if (err) + goto out; + +retry: + /* + * Used to check for invalidations in progress, of the pfn that is + * returned by pfn_to_pfn_prot below. + */ + mmu_seq = kvm->mmu_notifier_seq; + /* + * Ensure the read of mmu_notifier_seq isn't reordered with PTE reads in + * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't + * risk the page we get a reference to getting unmapped before we have a + * chance to grab the mmu_lock without mmu_notifier_retry() noticing. + * + * This smp_rmb() pairs with the effective smp_wmb() of the combination + * of the pte_unmap_unlock() after the PTE is zapped, and the + * spin_lock() in kvm_mmu_notifier_invalidate_() before + * mmu_notifier_seq is incremented. + */ + smp_rmb(); + + /* Slow path - ask KVM core whether we can access this GPA */ + pfn = gfn_to_pfn_prot(kvm, gfn, write, &writeable); + if (is_error_noslot_pfn(pfn)) { + err = -EFAULT; + goto out; + } + + spin_lock(&kvm->mmu_lock); + /* Check if an invalidation has taken place since we got pfn */ + if (mmu_notifier_retry(kvm, mmu_seq)) { + /* + * This can happen when mappings are changed asynchronously, but + * also synchronously if a COW is triggered by + * gfn_to_pfn_prot(). + */ + spin_unlock(&kvm->mmu_lock); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + if (retry_no > 100) { + retry_no = 0; + schedule(); + } + retry_no++; + goto retry; + } + + if (vma_pagesize == PAGE_SIZE && !force_pte) { + /* + * Only PMD_SIZE transparent hugepages(THP) are + * currently supported. This code will need to be + * updated to support other THP sizes. + * + * Make sure the host VA and the guest IPA are sufficiently + * aligned and that the block is contained within the memslot. + */ + ++vcpu->stat.huge_thp_exits; + if (fault_supports_huge_mapping(memslot, hva, write) && + transparent_hugepage_adjust(&pfn, &gpa)) { + ++vcpu->stat.huge_adjust_exits; + vma_pagesize = PMD_SIZE; + } + } + + /* Set up the prot bits */ + prot_bits = _PAGE_PRESENT | __READABLE; + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) + prot_bits |= _CACHE_SUC; + else + prot_bits |= _CACHE_CC; + + if (writeable) { + prot_bits |= _PAGE_WRITE; + if (write) { + prot_bits |= __WRITEABLE; + mark_page_dirty(kvm, gfn); + kvm_set_pfn_dirty(pfn); + } + } + + if (vma_pagesize == PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, __pgprot(prot_bits)); + new_pmd = pmd_mkhuge(new_pmd); + if (writeable && write) { + gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT; + for (i = 0; i < PTRS_PER_PTE; i++) + mark_page_dirty(kvm, base_gfn + i); + } + + ++vcpu->stat.huge_set_exits; + kvm_set_pmd_huge(vcpu, memcache, gpa, &new_pmd); + } else { + pte_t new_pte = pfn_pte(pfn, __pgprot(prot_bits)); + if (writeable && write) + mark_page_dirty(kvm, gfn); + + /* Ensure page tables are allocated */ + ptep = kvm_pte_for_gpa(kvm, memcache, gpa); + if (ptep && kvm_pte_huge(*ptep) && write) + ptep = kvm_split_huge(vcpu, ptep, gfn, vma, hva); + + set_pte(ptep, new_pte); + err = 0; + } + + spin_unlock(&kvm->mmu_lock); + kvm_release_pfn_clean(pfn); + kvm_set_pfn_accessed(pfn); +out: + srcu_read_unlock(&kvm->srcu, srcu_idx); + return err; +} + +int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, + bool write) +{ + int ret; + + ret = kvm_map_page(vcpu, badv, write); + if (ret) + return ret; + + /* Invalidate this entry in the TLB */ + return kvm_tlb_flush_gpa(vcpu, badv); +} + +/** + * kvm_flush_tlb_all() - Flush all root TLB entries for + * guests. + * + * Invalidate all entries including GVA-->GPA and GPA-->HPA mappings. + */ +void kvm_flush_tlb_all(void) +{ + unsigned long flags; + + local_irq_save(flags); + invtlb_all(INVTLB_ALLGID, 0, 0); + local_irq_restore(flags); +} diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c new file mode 100644 index 0000000000000000000000000000000000000000..6fa063434ae2f55c5447e2d0372851aad1af25a6 --- /dev/null +++ b/arch/loongarch/kvm/timer.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include "trace.h" +#include "kvm_compat.h" + +/* + * ktime_to_tick() - Scale ktime_t to a 64-bit stable timer. + * + * Caches the dynamic nanosecond bias in vcpu->arch.timer_dyn_bias. + */ +static u64 ktime_to_tick(struct kvm_vcpu *vcpu, ktime_t now) +{ + s64 now_ns, periods; + u64 delta; + + now_ns = ktime_to_ns(now); + delta = now_ns + vcpu->arch.timer_dyn_bias; + + if (delta >= vcpu->arch.timer_period) { + /* If delta is out of safe range the bias needs adjusting */ + periods = div64_s64(now_ns, vcpu->arch.timer_period); + vcpu->arch.timer_dyn_bias = -periods * vcpu->arch.timer_period; + /* Recalculate delta with new bias */ + delta = now_ns + vcpu->arch.timer_dyn_bias; + } + + /* + * We've ensured that: + * delta < timer_period + */ + return div_u64(delta * vcpu->arch.timer_mhz, MNSEC_PER_SEC); +} + +/** + * kvm_resume_hrtimer() - Resume hrtimer, updating expiry. + * @vcpu: Virtual CPU. + * @now: ktime at point of resume. + * @stable_timer: stable timer at point of resume. + * + * Resumes the timer and updates the timer expiry based on @now and @count. + */ +static void kvm_resume_hrtimer(struct kvm_vcpu *vcpu, ktime_t now, u64 stable_timer) +{ + u64 delta; + ktime_t expire; + + /* Stable timer decreased to zero or + * initialize to zero, set 4 second timer + */ + delta = div_u64(stable_timer * MNSEC_PER_SEC, vcpu->arch.timer_mhz); + expire = ktime_add_ns(now, delta); + + /* Update hrtimer to use new timeout */ + hrtimer_cancel(&vcpu->arch.swtimer); + hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED); +} + +/** + * kvm_init_timer() - Initialise stable timer. + * @vcpu: Virtual CPU. + * @timer_hz: Frequency of timer. + * + * Initialise the timer to the specified frequency, zero it, and set it going if + * it's enabled. + */ +void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz) +{ + ktime_t now; + unsigned long ticks; + struct loongarch_csrs *csr = vcpu->arch.csr; + + vcpu->arch.timer_mhz = timer_hz >> 20; + vcpu->arch.timer_period = div_u64((u64)MNSEC_PER_SEC * IOCSR_TIMER_MASK, vcpu->arch.timer_mhz); + vcpu->arch.timer_dyn_bias = 0; + + /* Starting at 0 */ + ticks = 0; + now = ktime_get(); + vcpu->arch.timer_bias = ticks - ktime_to_tick(vcpu, now); + vcpu->arch.timer_bias &= IOCSR_TIMER_MASK; + + kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, ticks); +} + +/** + * kvm_count_timeout() - Push timer forward on timeout. + * @vcpu: Virtual CPU. + * + * Handle an hrtimer event by push the hrtimer forward a period. + * + * Returns: The hrtimer_restart value to return to the hrtimer subsystem. + */ +enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu) +{ + unsigned long timer_cfg; + + /* Add the Count period to the current expiry time */ + timer_cfg = kvm_read_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG); + if (timer_cfg & KVM_TCFG_PERIOD) { + hrtimer_add_expires_ns(&vcpu->arch.swtimer, timer_cfg & KVM_TCFG_VAL); + return HRTIMER_RESTART; + } else + return HRTIMER_NORESTART; +} + +/* + * kvm_restore_timer() - Restore timer state. + * @vcpu: Virtual CPU. + * + * Restore soft timer state from saved context. + */ +void kvm_restore_timer(struct kvm_vcpu *vcpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + ktime_t saved_ktime, now; + u64 stable_timer, new_timertick = 0; + u64 delta = 0; + int expired = 0; + unsigned long timer_cfg; + + /* + * Set guest stable timer cfg csr + */ + timer_cfg = kvm_read_sw_gcsr(csr, KVM_CSR_TCFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_ESTAT); + if (!(timer_cfg & KVM_TCFG_EN)) { + kvm_restore_hw_gcsr(csr, KVM_CSR_TCFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_TVAL); + return; + } + + now = ktime_get(); + saved_ktime = vcpu->arch.stable_ktime_saved; + stable_timer = kvm_read_sw_gcsr(csr, KVM_CSR_TVAL); + + /*hrtimer not expire */ + delta = ktime_to_tick(vcpu, ktime_sub(now, saved_ktime)); + if (delta >= stable_timer) + expired = 1; + + if (expired) { + if (timer_cfg & KVM_TCFG_PERIOD) { + new_timertick = (delta - stable_timer) % (timer_cfg & KVM_TCFG_VAL); + } else { + new_timertick = 1; + } + } else { + new_timertick = stable_timer - delta; + } + + new_timertick &= KVM_TCFG_VAL; + kvm_write_gcsr_timercfg(timer_cfg); + kvm_write_gcsr_timertick(new_timertick); + if (expired) + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); +} + +/* + * kvm_acquire_timer() - Switch to hard timer state. + * @vcpu: Virtual CPU. + * + * Restore hard timer state on top of existing soft timer state if possible. + * + * Since hard timer won't remain active over preemption, preemption should be + * disabled by the caller. + */ +void kvm_acquire_timer(struct kvm_vcpu *vcpu) +{ + unsigned long flags, guestcfg; + + guestcfg = kvm_read_csr_gcfg(); + if (!(guestcfg & KVM_GCFG_TIT)) + return; + + /* enable guest access to hard timer */ + kvm_write_csr_gcfg(guestcfg & ~KVM_GCFG_TIT); + + /* + * Freeze the soft-timer and sync the guest stable timer with it. We do + * this with interrupts disabled to avoid latency. + */ + local_irq_save(flags); + hrtimer_cancel(&vcpu->arch.swtimer); + local_irq_restore(flags); +} + + +/* + * _kvm_save_timer() - Switch to software emulation of guest timer. + * @vcpu: Virtual CPU. + * + * Save guest timer state and switch to software emulation of guest + * timer. The hard timer must already be in use, so preemption should be + * disabled. + */ +static ktime_t _kvm_save_timer(struct kvm_vcpu *vcpu, u64 *stable_timer) +{ + u64 end_stable_timer; + ktime_t before_time; + + before_time = ktime_get(); + + /* + * Record a final stable timer which we will transfer to the soft-timer. + */ + end_stable_timer = kvm_read_gcsr_timertick(); + *stable_timer = end_stable_timer; + + kvm_resume_hrtimer(vcpu, before_time, end_stable_timer); + return before_time; +} + +/* + * kvm_save_timer() - Save guest timer state. + * @vcpu: Virtual CPU. + * + * Save guest timer state and switch to soft guest timer if hard timer was in + * use. + */ +void kvm_save_timer(struct kvm_vcpu *vcpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + unsigned long guestcfg; + u64 stable_timer = 0; + ktime_t save_ktime; + + preempt_disable(); + guestcfg = kvm_read_csr_gcfg(); + if (!(guestcfg & KVM_GCFG_TIT)) { + /* disable guest use of hard timer */ + kvm_write_csr_gcfg(guestcfg | KVM_GCFG_TIT); + + /* save hard timer state */ + kvm_save_hw_gcsr(csr, KVM_CSR_TCFG); + if (kvm_read_sw_gcsr(csr, KVM_CSR_TCFG) & KVM_TCFG_EN) { + save_ktime = _kvm_save_timer(vcpu, &stable_timer); + kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, stable_timer); + vcpu->arch.stable_ktime_saved = save_ktime; + if (stable_timer == IOCSR_TIMER_MASK) + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); + } else { + kvm_save_hw_gcsr(csr, KVM_CSR_TVAL); + } + } + + /* save timer-related state to VCPU context */ + kvm_save_hw_gcsr(csr, KVM_CSR_ESTAT); + preempt_enable(); +} + +void kvm_reset_timer(struct kvm_vcpu *vcpu) +{ + kvm_write_gcsr_timercfg(0); + kvm_write_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG, 0); + hrtimer_cancel(&vcpu->arch.swtimer); +} diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h new file mode 100644 index 0000000000000000000000000000000000000000..b38a5ebe0f8b2bd1dc0dc6ba67621dd96cd6511a --- /dev/null +++ b/arch/loongarch/kvm/trace.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_KVM_H + +#include +#include "kvm_compat.h" +#include "kvmcsr.h" + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace + +/* + * arch/loongarch/kvm/loongarch.c + */ +extern bool kvm_trace_guest_mode_change; +int kvm_guest_mode_change_trace_reg(void); +void kvm_guest_mode_change_trace_unreg(void); + +/* + * Tracepoints for VM enters + */ +DECLARE_EVENT_CLASS(kvm_transition, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + TP_STRUCT__entry( + __field(unsigned long, pc) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + ), + + TP_printk("PC: 0x%08lx", + __entry->pc) +); + +DEFINE_EVENT(kvm_transition, kvm_enter, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +DEFINE_EVENT(kvm_transition, kvm_reenter, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +DEFINE_EVENT(kvm_transition, kvm_out, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +/* The first 32 exit reasons correspond to Cause.ExcCode */ +#define KVM_TRACE_EXIT_INT 0 +#define KVM_TRACE_EXIT_TLBLD (EXCCODE_TLBL) +#define KVM_TRACE_EXIT_TLBST (EXCCODE_TLBS) +#define KVM_TRACE_EXIT_TLBI (EXCCODE_TLBI) +#define KVM_TRACE_EXIT_TLBMOD (EXCCODE_TLBM) +#define KVM_TRACE_EXIT_TLBNR (EXCCODE_TLBNR) +#define KVM_TRACE_EXIT_TLBNX (EXCCODE_TLBNX) +#define KVM_TRACE_EXIT_TLBPE (EXCCODE_TLBPE) +#define KVM_TRACE_EXIT_ADDE (EXCCODE_ADE) +#define KVM_TRACE_EXIT_UNALIGN (EXCCODE_ALE) +#define KVM_TRACE_EXIT_ODB (EXCCODE_BCE) +#define KVM_TRACE_EXIT_SYSCALL (EXCCODE_SYS) +#define KVM_TRACE_EXIT_BP (EXCCODE_BP) +#define KVM_TRACE_EXIT_INE (EXCCODE_INE) +#define KVM_TRACE_EXIT_IPE (EXCCODE_IPE) +#define KVM_TRACE_EXIT_FPDIS (EXCCODE_FPDIS) +#define KVM_TRACE_EXIT_LSXDIS (EXCCODE_LSXDIS) +#define KVM_TRACE_EXIT_LASXDIS (EXCCODE_LASXDIS) +#define KVM_TRACE_EXIT_FPE (EXCCODE_FPE) +#define KVM_TRACE_EXIT_WATCH (EXCCODE_WATCH) +#define KVM_TRACE_EXIT_GSPR (EXCCODE_GSPR) +#define KVM_TRACE_EXIT_HC (EXCCODE_HVC) +#define KVM_TRACE_EXIT_GCM (EXCCODE_GCM) + +/* Further exit reasons */ +#define KVM_TRACE_EXIT_IDLE 64 +#define KVM_TRACE_EXIT_CACHE 65 +#define KVM_TRACE_EXIT_SIGNAL 66 + +/* Tracepoints for VM exits */ +#define kvm_trace_symbol_exit_types \ + { KVM_TRACE_EXIT_INT, "Interrupt" }, \ + { KVM_TRACE_EXIT_TLBLD, "TLB (LD)" }, \ + { KVM_TRACE_EXIT_TLBST, "TLB (ST)" }, \ + { KVM_TRACE_EXIT_TLBI, "TLB Ifetch" }, \ + { KVM_TRACE_EXIT_TLBMOD, "TLB Mod" }, \ + { KVM_TRACE_EXIT_TLBNR, "TLB RI" }, \ + { KVM_TRACE_EXIT_TLBNX, "TLB XI" }, \ + { KVM_TRACE_EXIT_TLBPE, "TLB Previlege Error" },\ + { KVM_TRACE_EXIT_ADDE, "Address Error" }, \ + { KVM_TRACE_EXIT_UNALIGN, "Address unalign" }, \ + { KVM_TRACE_EXIT_ODB, "Out boundary" }, \ + { KVM_TRACE_EXIT_SYSCALL, "System Call" }, \ + { KVM_TRACE_EXIT_BP, "Breakpoint" }, \ + { KVM_TRACE_EXIT_INE, "Reserved Inst" }, \ + { KVM_TRACE_EXIT_IPE, "Inst prev error" }, \ + { KVM_TRACE_EXIT_FPDIS, "FPU disable" }, \ + { KVM_TRACE_EXIT_LSXDIS, "LSX disable" }, \ + { KVM_TRACE_EXIT_LASXDIS, "LASX disable" }, \ + { KVM_TRACE_EXIT_FPE, "FPE" }, \ + { KVM_TRACE_EXIT_WATCH, "DEBUG" }, \ + { KVM_TRACE_EXIT_GSPR, "GSPR" }, \ + { KVM_TRACE_EXIT_HC, "Hypercall" }, \ + { KVM_TRACE_EXIT_GCM, "CSR Mod" }, \ + { KVM_TRACE_EXIT_IDLE, "IDLE" }, \ + { KVM_TRACE_EXIT_CACHE, "CACHE" }, \ + { KVM_TRACE_EXIT_SIGNAL, "Signal" } + +TRACE_EVENT(kvm_exit, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason), + TP_ARGS(vcpu, reason), + TP_STRUCT__entry( + __field(unsigned long, pc) + __field(unsigned int, reason) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + __entry->reason = reason; + ), + + TP_printk("[%s]PC: 0x%08lx", + __print_symbolic(__entry->reason, + kvm_trace_symbol_exit_types), + __entry->pc) +); + +#define KVM_TRACE_AUX_RESTORE 0 +#define KVM_TRACE_AUX_SAVE 1 +#define KVM_TRACE_AUX_ENABLE 2 +#define KVM_TRACE_AUX_DISABLE 3 +#define KVM_TRACE_AUX_DISCARD 4 + +#define KVM_TRACE_AUX_FPU 1 +#define KVM_TRACE_AUX_LSX 2 +#define KVM_TRACE_AUX_FPU_LSX 3 +#define KVM_TRACE_AUX_LASX 4 +#define KVM_TRACE_AUX_FPU_LSX_LASX 7 + +#define kvm_trace_symbol_aux_op \ + { KVM_TRACE_AUX_RESTORE, "restore" }, \ + { KVM_TRACE_AUX_SAVE, "save" }, \ + { KVM_TRACE_AUX_ENABLE, "enable" }, \ + { KVM_TRACE_AUX_DISABLE, "disable" }, \ + { KVM_TRACE_AUX_DISCARD, "discard" } + +#define kvm_trace_symbol_aux_state \ + { KVM_TRACE_AUX_FPU, "FPU" }, \ + { KVM_TRACE_AUX_LSX, "LSX" }, \ + { KVM_TRACE_AUX_LASX, "LASX" }, \ + { KVM_TRACE_AUX_FPU_LSX, "FPU & LSX" }, \ + { KVM_TRACE_AUX_FPU_LSX_LASX, "FPU & LSX & LASX" } + +TRACE_EVENT(kvm_aux, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op, + unsigned int state), + TP_ARGS(vcpu, op, state), + TP_STRUCT__entry( + __field(unsigned long, pc) + __field(u8, op) + __field(u8, state) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + __entry->op = op; + __entry->state = state; + ), + + TP_printk("%s %s PC: 0x%08lx", + __print_symbolic(__entry->op, + kvm_trace_symbol_aux_op), + __print_symbolic(__entry->state, + kvm_trace_symbol_aux_state), + __entry->pc) +); + +TRACE_EVENT(kvm_vpid_change, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned long vpid), + TP_ARGS(vcpu, vpid), + TP_STRUCT__entry( + __field(unsigned long, vpid) + ), + + TP_fast_assign( + __entry->vpid = vpid; + ), + + TP_printk("vpid: 0x%08lx", + __entry->vpid) +); + +#endif /* _TRACE_KVM_H */ + +/* This part must be outside protection */ +#include diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b2759f2a04d55a61e82d87d5b9ad1e656d63056e --- /dev/null +++ b/arch/loongarch/lib/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for LoongArch-specific library files.. +# + +lib-y += delay.o memset.o memcpy.o memmove.o \ + clear_user.o copy_user.o csum.o unaligned.o dump_tlb.o + +obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o diff --git a/arch/loongarch/lib/clear_user.S b/arch/loongarch/lib/clear_user.S new file mode 100644 index 0000000000000000000000000000000000000000..248121a23ecd2eab7650905c99ce4bf257181838 --- /dev/null +++ b/arch/loongarch/lib/clear_user.S @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +SYM_FUNC_START(__clear_user) + /* + * Some CPUs support hardware unaligned access + */ + ALTERNATIVE "b __clear_user_generic", \ + "b __clear_user_fast", CPU_FEATURE_UAL +SYM_FUNC_END(__clear_user) + +EXPORT_SYMBOL(__clear_user) + +/* + * unsigned long __clear_user_generic(void *addr, size_t size) + * + * a0: addr + * a1: size + */ +SYM_FUNC_START(__clear_user_generic) + beqz a1, 2f + +1: st.b zero, a0, 0 + addi.d a0, a0, 1 + addi.d a1, a1, -1 + bgt a1, zero, 1b + +2: move a0, a1 + jr ra + + _asm_extable 1b, 2b +SYM_FUNC_END(__clear_user_generic) + +/* + * unsigned long __clear_user_fast(void *addr, unsigned long size) + * + * a0: addr + * a1: size + */ +SYM_FUNC_START(__clear_user_fast) + sltui t0, a1, 9 + bnez t0, .Lsmall + + add.d a2, a0, a1 +0: st.d zero, a0, 0 + + /* align up address */ + addi.d a0, a0, 8 + bstrins.d a0, zero, 2, 0 + + addi.d a3, a2, -64 + bgeu a0, a3, .Llt64 + + /* set 64 bytes at a time */ +.Lloop64: +1: st.d zero, a0, 0 +2: st.d zero, a0, 8 +3: st.d zero, a0, 16 +4: st.d zero, a0, 24 +5: st.d zero, a0, 32 +6: st.d zero, a0, 40 +7: st.d zero, a0, 48 +8: st.d zero, a0, 56 + addi.d a0, a0, 64 + bltu a0, a3, .Lloop64 + + /* set the remaining bytes */ +.Llt64: + addi.d a3, a2, -32 + bgeu a0, a3, .Llt32 +9: st.d zero, a0, 0 +10: st.d zero, a0, 8 +11: st.d zero, a0, 16 +12: st.d zero, a0, 24 + addi.d a0, a0, 32 + +.Llt32: + addi.d a3, a2, -16 + bgeu a0, a3, .Llt16 +13: st.d zero, a0, 0 +14: st.d zero, a0, 8 + addi.d a0, a0, 16 + +.Llt16: + addi.d a3, a2, -8 + bgeu a0, a3, .Llt8 +15: st.d zero, a0, 0 + addi.d a0, a0, 8 + +.Llt8: +16: st.d zero, a2, -8 + + /* return */ + move a0, zero + jr ra + + .align 4 +.Lsmall: + pcaddi t0, 4 + slli.d a2, a1, 4 + add.d t0, t0, a2 + jr t0 + + .align 4 + move a0, zero + jr ra + + .align 4 +17: st.b zero, a0, 0 + move a0, zero + jr ra + + .align 4 +18: st.h zero, a0, 0 + move a0, zero + jr ra + + .align 4 +19: st.h zero, a0, 0 +20: st.b zero, a0, 2 + move a0, zero + jr ra + + .align 4 +21: st.w zero, a0, 0 + move a0, zero + jr ra + + .align 4 +22: st.w zero, a0, 0 +23: st.b zero, a0, 4 + move a0, zero + jr ra + + .align 4 +24: st.w zero, a0, 0 +25: st.h zero, a0, 4 + move a0, zero + jr ra + + .align 4 +26: st.w zero, a0, 0 +27: st.w zero, a0, 3 + move a0, zero + jr ra + + .align 4 +28: st.d zero, a0, 0 + move a0, zero + jr ra + + /* fixup and ex_table */ +.Llarge_fixup: + sub.d a1, a2, a0 + +.Lsmall_fixup: +29: st.b zero, a0, 0 + addi.d a0, a0, 1 + addi.d a1, a1, -1 + bgt a1, zero, 29b + +.Lexit: + move a0, a1 + jr ra + + _asm_extable 0b, .Lsmall_fixup + _asm_extable 1b, .Llarge_fixup + _asm_extable 2b, .Llarge_fixup + _asm_extable 3b, .Llarge_fixup + _asm_extable 4b, .Llarge_fixup + _asm_extable 5b, .Llarge_fixup + _asm_extable 6b, .Llarge_fixup + _asm_extable 7b, .Llarge_fixup + _asm_extable 8b, .Llarge_fixup + _asm_extable 9b, .Llarge_fixup + _asm_extable 10b, .Llarge_fixup + _asm_extable 11b, .Llarge_fixup + _asm_extable 12b, .Llarge_fixup + _asm_extable 13b, .Llarge_fixup + _asm_extable 14b, .Llarge_fixup + _asm_extable 15b, .Llarge_fixup + _asm_extable 16b, .Llarge_fixup + _asm_extable 17b, .Lexit + _asm_extable 18b, .Lsmall_fixup + _asm_extable 19b, .Lsmall_fixup + _asm_extable 20b, .Lsmall_fixup + _asm_extable 21b, .Lsmall_fixup + _asm_extable 22b, .Lsmall_fixup + _asm_extable 23b, .Lsmall_fixup + _asm_extable 24b, .Lsmall_fixup + _asm_extable 25b, .Lsmall_fixup + _asm_extable 26b, .Lsmall_fixup + _asm_extable 27b, .Lsmall_fixup + _asm_extable 28b, .Lsmall_fixup + _asm_extable 29b, .Lexit +SYM_FUNC_END(__clear_user_fast) diff --git a/arch/loongarch/lib/copy_user.S b/arch/loongarch/lib/copy_user.S new file mode 100644 index 0000000000000000000000000000000000000000..58cfc6285d4658a6e7052876de93193b76fc5599 --- /dev/null +++ b/arch/loongarch/lib/copy_user.S @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +SYM_FUNC_START(__copy_user) + /* + * Some CPUs support hardware unaligned access + */ + ALTERNATIVE "b __copy_user_generic", \ + "b __copy_user_fast", CPU_FEATURE_UAL +SYM_FUNC_END(__copy_user) + +EXPORT_SYMBOL(__copy_user) + +/* + * unsigned long __copy_user_generic(void *to, const void *from, size_t n) + * + * a0: to + * a1: from + * a2: n + */ +SYM_FUNC_START(__copy_user_generic) + beqz a2, 3f + +1: ld.b t0, a1, 0 +2: st.b t0, a0, 0 + addi.d a0, a0, 1 + addi.d a1, a1, 1 + addi.d a2, a2, -1 + bgt a2, zero, 1b + +3: move a0, a2 + jr ra + + _asm_extable 1b, 3b + _asm_extable 2b, 3b +SYM_FUNC_END(__copy_user_generic) + +/* + * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n) + * + * a0: to + * a1: from + * a2: n + */ +SYM_FUNC_START(__copy_user_fast) + sltui t0, a2, 9 + bnez t0, .Lsmall + +0: ld.d t0, a1, 0 +1: st.d t0, a0, 0 + add.d a3, a1, a2 + add.d a2, a0, a2 + + /* align up destination address */ + andi t1, a0, 7 + sub.d t0, zero, t1 + addi.d t0, t0, 8 + add.d a1, a1, t0 + add.d a0, a0, t0 + + addi.d a4, a3, -64 + bgeu a1, a4, .Llt64 + + /* copy 64 bytes at a time */ +.Lloop64: +2: ld.d t0, a1, 0 +3: ld.d t1, a1, 8 +4: ld.d t2, a1, 16 +5: ld.d t3, a1, 24 +6: ld.d t4, a1, 32 +7: ld.d t5, a1, 40 +8: ld.d t6, a1, 48 +9: ld.d t7, a1, 56 +10: st.d t0, a0, 0 +11: st.d t1, a0, 8 +12: st.d t2, a0, 16 +13: st.d t3, a0, 24 +14: st.d t4, a0, 32 +15: st.d t5, a0, 40 +16: st.d t6, a0, 48 +17: st.d t7, a0, 56 + addi.d a1, a1, 64 + addi.d a0, a0, 64 + bltu a1, a4, .Lloop64 + + /* copy the remaining bytes */ +.Llt64: + addi.d a4, a3, -32 + bgeu a1, a4, .Llt32 +18: ld.d t0, a1, 0 +19: ld.d t1, a1, 8 +20: ld.d t2, a1, 16 +21: ld.d t3, a1, 24 +22: st.d t0, a0, 0 +23: st.d t1, a0, 8 +24: st.d t2, a0, 16 +25: st.d t3, a0, 24 + addi.d a1, a1, 32 + addi.d a0, a0, 32 + +.Llt32: + addi.d a4, a3, -16 + bgeu a1, a4, .Llt16 +26: ld.d t0, a1, 0 +27: ld.d t1, a1, 8 +28: st.d t0, a0, 0 +29: st.d t1, a0, 8 + addi.d a1, a1, 16 + addi.d a0, a0, 16 + +.Llt16: + addi.d a4, a3, -8 + bgeu a1, a4, .Llt8 +30: ld.d t0, a1, 0 +31: st.d t0, a0, 0 + addi.d a1, a1, 8 + addi.d a0, a0, 8 + +.Llt8: +32: ld.d t0, a3, -8 +33: st.d t0, a2, -8 + + /* return */ + move a0, zero + jr ra + + .align 5 +.Lsmall: + pcaddi t0, 8 + slli.d a3, a2, 5 + add.d t0, t0, a3 + jr t0 + + .align 5 + move a0, zero + jr ra + + .align 5 +34: ld.b t0, a1, 0 +35: st.b t0, a0, 0 + move a0, zero + jr ra + + .align 5 +36: ld.h t0, a1, 0 +37: st.h t0, a0, 0 + move a0, zero + jr ra + + .align 5 +38: ld.h t0, a1, 0 +39: ld.b t1, a1, 2 +40: st.h t0, a0, 0 +41: st.b t1, a0, 2 + move a0, zero + jr ra + + .align 5 +42: ld.w t0, a1, 0 +43: st.w t0, a0, 0 + move a0, zero + jr ra + + .align 5 +44: ld.w t0, a1, 0 +45: ld.b t1, a1, 4 +46: st.w t0, a0, 0 +47: st.b t1, a0, 4 + move a0, zero + jr ra + + .align 5 +48: ld.w t0, a1, 0 +49: ld.h t1, a1, 4 +50: st.w t0, a0, 0 +51: st.h t1, a0, 4 + move a0, zero + jr ra + + .align 5 +52: ld.w t0, a1, 0 +53: ld.w t1, a1, 3 +54: st.w t0, a0, 0 +55: st.w t1, a0, 3 + move a0, zero + jr ra + + .align 5 +56: ld.d t0, a1, 0 +57: st.d t0, a0, 0 + move a0, zero + jr ra + + /* fixup and ex_table */ +.Llarge_fixup: + sub.d a2, a2, a0 + +.Lsmall_fixup: +58: ld.b t0, a1, 0 +59: st.b t0, a0, 0 + addi.d a0, a0, 1 + addi.d a1, a1, 1 + addi.d a2, a2, -1 + bgt a2, zero, 58b + +.Lexit: + move a0, a2 + jr ra + + _asm_extable 0b, .Lsmall_fixup + _asm_extable 1b, .Lsmall_fixup + _asm_extable 2b, .Llarge_fixup + _asm_extable 3b, .Llarge_fixup + _asm_extable 4b, .Llarge_fixup + _asm_extable 5b, .Llarge_fixup + _asm_extable 6b, .Llarge_fixup + _asm_extable 7b, .Llarge_fixup + _asm_extable 8b, .Llarge_fixup + _asm_extable 9b, .Llarge_fixup + _asm_extable 10b, .Llarge_fixup + _asm_extable 11b, .Llarge_fixup + _asm_extable 12b, .Llarge_fixup + _asm_extable 13b, .Llarge_fixup + _asm_extable 14b, .Llarge_fixup + _asm_extable 15b, .Llarge_fixup + _asm_extable 16b, .Llarge_fixup + _asm_extable 17b, .Llarge_fixup + _asm_extable 18b, .Llarge_fixup + _asm_extable 19b, .Llarge_fixup + _asm_extable 20b, .Llarge_fixup + _asm_extable 21b, .Llarge_fixup + _asm_extable 22b, .Llarge_fixup + _asm_extable 23b, .Llarge_fixup + _asm_extable 24b, .Llarge_fixup + _asm_extable 25b, .Llarge_fixup + _asm_extable 26b, .Llarge_fixup + _asm_extable 27b, .Llarge_fixup + _asm_extable 28b, .Llarge_fixup + _asm_extable 29b, .Llarge_fixup + _asm_extable 30b, .Llarge_fixup + _asm_extable 31b, .Llarge_fixup + _asm_extable 32b, .Llarge_fixup + _asm_extable 33b, .Llarge_fixup + _asm_extable 34b, .Lexit + _asm_extable 35b, .Lexit + _asm_extable 36b, .Lsmall_fixup + _asm_extable 37b, .Lsmall_fixup + _asm_extable 38b, .Lsmall_fixup + _asm_extable 39b, .Lsmall_fixup + _asm_extable 40b, .Lsmall_fixup + _asm_extable 41b, .Lsmall_fixup + _asm_extable 42b, .Lsmall_fixup + _asm_extable 43b, .Lsmall_fixup + _asm_extable 44b, .Lsmall_fixup + _asm_extable 45b, .Lsmall_fixup + _asm_extable 46b, .Lsmall_fixup + _asm_extable 47b, .Lsmall_fixup + _asm_extable 48b, .Lsmall_fixup + _asm_extable 49b, .Lsmall_fixup + _asm_extable 50b, .Lsmall_fixup + _asm_extable 51b, .Lsmall_fixup + _asm_extable 52b, .Lsmall_fixup + _asm_extable 53b, .Lsmall_fixup + _asm_extable 54b, .Lsmall_fixup + _asm_extable 55b, .Lsmall_fixup + _asm_extable 56b, .Lsmall_fixup + _asm_extable 57b, .Lsmall_fixup + _asm_extable 58b, .Lexit + _asm_extable 59b, .Lexit +SYM_FUNC_END(__copy_user_fast) diff --git a/arch/loongarch/lib/csum.c b/arch/loongarch/lib/csum.c new file mode 100644 index 0000000000000000000000000000000000000000..a5e84b403c3b34edfb4cf9ebfd12a88103c5d4fc --- /dev/null +++ b/arch/loongarch/lib/csum.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019-2020 Arm Ltd. + +#include +#include +#include + +#include + +static u64 accumulate(u64 sum, u64 data) +{ + sum += data; + if (sum < data) + sum += 1; + return sum; +} + +/* + * We over-read the buffer and this makes KASAN unhappy. Instead, disable + * instrumentation and call kasan explicitly. + */ +unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len) +{ + unsigned int offset, shift, sum; + const u64 *ptr; + u64 data, sum64 = 0; + + if (unlikely(len == 0)) + return 0; + + offset = (unsigned long)buff & 7; + /* + * This is to all intents and purposes safe, since rounding down cannot + * result in a different page or cache line being accessed, and @buff + * should absolutely not be pointing to anything read-sensitive. We do, + * however, have to be careful not to piss off KASAN, which means using + * unchecked reads to accommodate the head and tail, for which we'll + * compensate with an explicit check up-front. + */ + kasan_check_read(buff, len); + ptr = (u64 *)(buff - offset); + len = len + offset - 8; + + /* + * Head: zero out any excess leading bytes. Shifting back by the same + * amount should be at least as fast as any other way of handling the + * odd/even alignment, and means we can ignore it until the very end. + */ + shift = offset * 8; + data = *ptr++; + data = (data >> shift) << shift; + + /* + * Body: straightforward aligned loads from here on (the paired loads + * underlying the quadword type still only need dword alignment). The + * main loop strictly excludes the tail, so the second loop will always + * run at least once. + */ + while (unlikely(len > 64)) { + __uint128_t tmp1, tmp2, tmp3, tmp4; + + tmp1 = *(__uint128_t *)ptr; + tmp2 = *(__uint128_t *)(ptr + 2); + tmp3 = *(__uint128_t *)(ptr + 4); + tmp4 = *(__uint128_t *)(ptr + 6); + + len -= 64; + ptr += 8; + + /* This is the "don't dump the carry flag into a GPR" idiom */ + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp2 += (tmp2 >> 64) | (tmp2 << 64); + tmp3 += (tmp3 >> 64) | (tmp3 << 64); + tmp4 += (tmp4 >> 64) | (tmp4 << 64); + tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64); + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64); + tmp3 += (tmp3 >> 64) | (tmp3 << 64); + tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64); + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + tmp1 = ((tmp1 >> 64) << 64) | sum64; + tmp1 += (tmp1 >> 64) | (tmp1 << 64); + sum64 = tmp1 >> 64; + } + while (len > 8) { + __uint128_t tmp; + + sum64 = accumulate(sum64, data); + tmp = *(__uint128_t *)ptr; + + len -= 16; + ptr += 2; + + data = tmp >> 64; + sum64 = accumulate(sum64, tmp); + } + if (len > 0) { + sum64 = accumulate(sum64, data); + data = *ptr; + len -= 8; + } + /* + * Tail: zero any over-read bytes similarly to the head, again + * preserving odd/even alignment. + */ + shift = len * -8; + data = (data << shift) >> shift; + sum64 = accumulate(sum64, data); + + /* Finally, folding */ + sum64 += (sum64 >> 32) | (sum64 << 32); + sum = sum64 >> 32; + sum += (sum >> 16) | (sum << 16); + if (offset & 1) + return (u16)swab32(sum); + + return sum >> 16; +} + +__sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum csum) +{ + __uint128_t src, dst; + u64 sum = (__force u64)csum; + + src = *(const __uint128_t *)saddr->s6_addr; + dst = *(const __uint128_t *)daddr->s6_addr; + + sum += (__force u32)htonl(len); + sum += (u32)proto << 24; + src += (src >> 64) | (src << 64); + dst += (dst >> 64) | (dst << 64); + + sum = accumulate(sum, src >> 64); + sum = accumulate(sum, dst >> 64); + + sum += ((sum >> 32) | (sum << 32)); + return csum_fold((__force __wsum)(sum >> 32)); +} +EXPORT_SYMBOL(csum_ipv6_magic); diff --git a/arch/loongarch/lib/delay.c b/arch/loongarch/lib/delay.c new file mode 100644 index 0000000000000000000000000000000000000000..208f274b9dbacd8dde2ee18746938455d447e021 --- /dev/null +++ b/arch/loongarch/lib/delay.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +#include + +void __delay(unsigned long cycles) +{ + u64 t0 = get_cycles(); + + while ((unsigned long)(get_cycles() - t0) < cycles) + cpu_relax(); +} +EXPORT_SYMBOL(__delay); + +/* + * Division by multiplication: you don't have to worry about + * loss of precision. + * + * Use only for very small delays ( < 1 msec). Should probably use a + * lookup table, really, as the multiplications take much too long with + * short delays. This is a "reasonable" implementation, though (and the + * first constant multiplications gets optimized away if the delay is + * a constant) + */ + +void __udelay(unsigned long us) +{ + __delay((us * 0x000010c7ull * HZ * lpj_fine) >> 32); +} +EXPORT_SYMBOL(__udelay); + +void __ndelay(unsigned long ns) +{ + __delay((ns * 0x00000005ull * HZ * lpj_fine) >> 32); +} +EXPORT_SYMBOL(__ndelay); diff --git a/arch/loongarch/lib/dump_tlb.c b/arch/loongarch/lib/dump_tlb.c new file mode 100644 index 0000000000000000000000000000000000000000..6a0e1c04ba91de3bc1c79b1c570568ad060fb51c --- /dev/null +++ b/arch/loongarch/lib/dump_tlb.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include + +#include +#include +#include +#include + +void dump_tlb_regs(void) +{ + const int field = 2 * sizeof(unsigned long); + + pr_info("Index : 0x%0x\n", read_csr_tlbidx()); + pr_info("PageSize : 0x%0x\n", read_csr_pagesize()); + pr_info("EntryHi : 0x%0*lx\n", field, read_csr_entryhi()); + pr_info("EntryLo0 : 0x%0*lx\n", field, read_csr_entrylo0()); + pr_info("EntryLo1 : 0x%0*lx\n", field, read_csr_entrylo1()); +} + +static void dump_tlb(int first, int last) +{ + unsigned long s_entryhi, entryhi, asid; + unsigned long long entrylo0, entrylo1, pa; + unsigned int index; + unsigned int s_index, s_asid; + unsigned int pagesize, c0, c1, i; + unsigned long asidmask = cpu_asid_mask(¤t_cpu_data); + int pwidth = 16; + int vwidth = 16; + int asidwidth = DIV_ROUND_UP(ilog2(asidmask) + 1, 4); + + s_entryhi = read_csr_entryhi(); + s_index = read_csr_tlbidx(); + s_asid = read_csr_asid(); + + for (i = first; i <= last; i++) { + write_csr_index(i); + tlb_read(); + pagesize = read_csr_pagesize(); + entryhi = read_csr_entryhi(); + entrylo0 = read_csr_entrylo0(); + entrylo1 = read_csr_entrylo1(); + index = read_csr_tlbidx(); + asid = read_csr_asid(); + + /* EHINV bit marks entire entry as invalid */ + if (index & CSR_TLBIDX_EHINV) + continue; + /* + * ASID takes effect in absence of G (global) bit. + */ + if (!((entrylo0 | entrylo1) & ENTRYLO_G) && + asid != s_asid) + continue; + + /* + * Only print entries in use + */ + printk("Index: %4d pgsize=0x%x ", i, (1 << pagesize)); + + c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; + c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; + + pr_cont("va=0x%0*lx asid=0x%0*lx", + vwidth, (entryhi & ~0x1fffUL), asidwidth, asid & asidmask); + + /* NR/NX are in awkward places, so mask them off separately */ + pa = entrylo0 & ~(ENTRYLO_NR | ENTRYLO_NX); + pa = pa & PAGE_MASK; + pr_cont("\n\t["); + pr_cont("nr=%d nx=%d ", + (entrylo0 & ENTRYLO_NR) ? 1 : 0, + (entrylo0 & ENTRYLO_NX) ? 1 : 0); + pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld] [", + pwidth, pa, c0, + (entrylo0 & ENTRYLO_D) ? 1 : 0, + (entrylo0 & ENTRYLO_V) ? 1 : 0, + (entrylo0 & ENTRYLO_G) ? 1 : 0, + (entrylo0 & ENTRYLO_PLV) >> ENTRYLO_PLV_SHIFT); + /* NR/NX are in awkward places, so mask them off separately */ + pa = entrylo1 & ~(ENTRYLO_NR | ENTRYLO_NX); + pa = pa & PAGE_MASK; + pr_cont("nr=%d nx=%d ", + (entrylo1 & ENTRYLO_NR) ? 1 : 0, + (entrylo1 & ENTRYLO_NX) ? 1 : 0); + pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld]\n", + pwidth, pa, c1, + (entrylo1 & ENTRYLO_D) ? 1 : 0, + (entrylo1 & ENTRYLO_V) ? 1 : 0, + (entrylo1 & ENTRYLO_G) ? 1 : 0, + (entrylo1 & ENTRYLO_PLV) >> ENTRYLO_PLV_SHIFT); + } + printk("\n"); + + write_csr_entryhi(s_entryhi); + write_csr_tlbidx(s_index); + write_csr_asid(s_asid); +} + +void dump_tlb_all(void) +{ + dump_tlb(0, current_cpu_data.tlbsize - 1); +} diff --git a/arch/loongarch/lib/memcpy.S b/arch/loongarch/lib/memcpy.S new file mode 100644 index 0000000000000000000000000000000000000000..86c1959e0397449f57c2f3118de8bdd6f72c07f5 --- /dev/null +++ b/arch/loongarch/lib/memcpy.S @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include + +SYM_FUNC_START_WEAK(memcpy) +SYM_FUNC_START_ALIAS(__memcpy) + /* + * Some CPUs support hardware unaligned access + */ + ALTERNATIVE "b __memcpy_generic", \ + "b __memcpy_fast", CPU_FEATURE_UAL +SYM_FUNC_END(memcpy) +SYM_FUNC_END_ALIAS(__memcpy) + +EXPORT_SYMBOL(memcpy) +EXPORT_SYMBOL(__memcpy) + +/* + * void *__memcpy_generic(void *dst, const void *src, size_t n) + * + * a0: dst + * a1: src + * a2: n + */ +SYM_FUNC_START(__memcpy_generic) + move a3, a0 + beqz a2, 2f + +1: ld.b t0, a1, 0 + st.b t0, a0, 0 + addi.d a0, a0, 1 + addi.d a1, a1, 1 + addi.d a2, a2, -1 + bgt a2, zero, 1b + +2: move a0, a3 + jr ra +SYM_FUNC_END(__memcpy_generic) + + .align 5 +SYM_FUNC_START_NOALIGN(__memcpy_small) + pcaddi t0, 8 + slli.d a2, a2, 5 + add.d t0, t0, a2 + jr t0 + + .align 5 +0: jr ra + + .align 5 +1: ld.b t0, a1, 0 + st.b t0, a0, 0 + jr ra + + .align 5 +2: ld.h t0, a1, 0 + st.h t0, a0, 0 + jr ra + + .align 5 +3: ld.h t0, a1, 0 + ld.b t1, a1, 2 + st.h t0, a0, 0 + st.b t1, a0, 2 + jr ra + + .align 5 +4: ld.w t0, a1, 0 + st.w t0, a0, 0 + jr ra + + .align 5 +5: ld.w t0, a1, 0 + ld.b t1, a1, 4 + st.w t0, a0, 0 + st.b t1, a0, 4 + jr ra + + .align 5 +6: ld.w t0, a1, 0 + ld.h t1, a1, 4 + st.w t0, a0, 0 + st.h t1, a0, 4 + jr ra + + .align 5 +7: ld.w t0, a1, 0 + ld.w t1, a1, 3 + st.w t0, a0, 0 + st.w t1, a0, 3 + jr ra + + .align 5 +8: ld.d t0, a1, 0 + st.d t0, a0, 0 + jr ra +SYM_FUNC_END(__memcpy_small) + +/* + * void *__memcpy_fast(void *dst, const void *src, size_t n) + * + * a0: dst + * a1: src + * a2: n + */ +SYM_FUNC_START(__memcpy_fast) + sltui t0, a2, 9 + bnez t0, __memcpy_small + + add.d a3, a1, a2 + add.d a2, a0, a2 + ld.d a6, a1, 0 + ld.d a7, a3, -8 + + /* align up destination address */ + andi t1, a0, 7 + sub.d t0, zero, t1 + addi.d t0, t0, 8 + add.d a1, a1, t0 + add.d a5, a0, t0 + + addi.d a4, a3, -64 + bgeu a1, a4, .Llt64 + + /* copy 64 bytes at a time */ +.Lloop64: + ld.d t0, a1, 0 + ld.d t1, a1, 8 + ld.d t2, a1, 16 + ld.d t3, a1, 24 + ld.d t4, a1, 32 + ld.d t5, a1, 40 + ld.d t6, a1, 48 + ld.d t7, a1, 56 + addi.d a1, a1, 64 + st.d t0, a5, 0 + st.d t1, a5, 8 + st.d t2, a5, 16 + st.d t3, a5, 24 + st.d t4, a5, 32 + st.d t5, a5, 40 + st.d t6, a5, 48 + st.d t7, a5, 56 + addi.d a5, a5, 64 + bltu a1, a4, .Lloop64 + + /* copy the remaining bytes */ +.Llt64: + addi.d a4, a3, -32 + bgeu a1, a4, .Llt32 + ld.d t0, a1, 0 + ld.d t1, a1, 8 + ld.d t2, a1, 16 + ld.d t3, a1, 24 + addi.d a1, a1, 32 + st.d t0, a5, 0 + st.d t1, a5, 8 + st.d t2, a5, 16 + st.d t3, a5, 24 + addi.d a5, a5, 32 + +.Llt32: + addi.d a4, a3, -16 + bgeu a1, a4, .Llt16 + ld.d t0, a1, 0 + ld.d t1, a1, 8 + addi.d a1, a1, 16 + st.d t0, a5, 0 + st.d t1, a5, 8 + addi.d a5, a5, 16 + +.Llt16: + addi.d a4, a3, -8 + bgeu a1, a4, .Llt8 + ld.d t0, a1, 0 + st.d t0, a5, 0 + +.Llt8: + st.d a6, a0, 0 + st.d a7, a2, -8 + + /* return */ + jr ra +SYM_FUNC_END(__memcpy_fast) diff --git a/arch/loongarch/lib/memmove.S b/arch/loongarch/lib/memmove.S new file mode 100644 index 0000000000000000000000000000000000000000..40b23b4f607050343adeb09f00e7506466853641 --- /dev/null +++ b/arch/loongarch/lib/memmove.S @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include + +SYM_FUNC_START_WEAK(memmove) +SYM_FUNC_START_ALIAS(__memmove) + blt a0, a1, __memcpy /* dst < src, memcpy */ + blt a1, a0, __rmemcpy /* src < dst, rmemcpy */ + jr ra /* dst == src, return */ +SYM_FUNC_END(memmove) +SYM_FUNC_END_ALIAS(__memmove) + +EXPORT_SYMBOL(memmove) +EXPORT_SYMBOL(__memmove) + +SYM_FUNC_START(__rmemcpy) + /* + * Some CPUs support hardware unaligned access + */ + ALTERNATIVE "b __rmemcpy_generic", \ + "b __rmemcpy_fast", CPU_FEATURE_UAL +SYM_FUNC_END(__rmemcpy) + +/* + * void *__rmemcpy_generic(void *dst, const void *src, size_t n) + * + * a0: dst + * a1: src + * a2: n + */ +SYM_FUNC_START(__rmemcpy_generic) + move a3, a0 + beqz a2, 2f + + add.d a0, a0, a2 + add.d a1, a1, a2 + +1: ld.b t0, a1, -1 + st.b t0, a0, -1 + addi.d a0, a0, -1 + addi.d a1, a1, -1 + addi.d a2, a2, -1 + bgt a2, zero, 1b + +2: move a0, a3 + jr ra +SYM_FUNC_END(__rmemcpy_generic) + +/* + * void *__rmemcpy_fast(void *dst, const void *src, size_t n) + * + * a0: dst + * a1: src + * a2: n + */ +SYM_FUNC_START(__rmemcpy_fast) + sltui t0, a2, 9 + bnez t0, __memcpy_small + + add.d a3, a1, a2 + add.d a2, a0, a2 + ld.d a6, a1, 0 + ld.d a7, a3, -8 + + /* align up destination address */ + andi t1, a2, 7 + sub.d a3, a3, t1 + sub.d a5, a2, t1 + + addi.d a4, a1, 64 + bgeu a4, a3, .Llt64 + + /* copy 64 bytes at a time */ +.Lloop64: + ld.d t0, a3, -8 + ld.d t1, a3, -16 + ld.d t2, a3, -24 + ld.d t3, a3, -32 + ld.d t4, a3, -40 + ld.d t5, a3, -48 + ld.d t6, a3, -56 + ld.d t7, a3, -64 + addi.d a3, a3, -64 + st.d t0, a5, -8 + st.d t1, a5, -16 + st.d t2, a5, -24 + st.d t3, a5, -32 + st.d t4, a5, -40 + st.d t5, a5, -48 + st.d t6, a5, -56 + st.d t7, a5, -64 + addi.d a5, a5, -64 + bltu a4, a3, .Lloop64 + + /* copy the remaining bytes */ +.Llt64: + addi.d a4, a1, 32 + bgeu a4, a3, .Llt32 + ld.d t0, a3, -8 + ld.d t1, a3, -16 + ld.d t2, a3, -24 + ld.d t3, a3, -32 + addi.d a3, a3, -32 + st.d t0, a5, -8 + st.d t1, a5, -16 + st.d t2, a5, -24 + st.d t3, a5, -32 + addi.d a5, a5, -32 + +.Llt32: + addi.d a4, a1, 16 + bgeu a4, a3, .Llt16 + ld.d t0, a3, -8 + ld.d t1, a3, -16 + addi.d a3, a3, -16 + st.d t0, a5, -8 + st.d t1, a5, -16 + addi.d a5, a5, -16 + +.Llt16: + addi.d a4, a1, 8 + bgeu a4, a3, .Llt8 + ld.d t0, a3, -8 + st.d t0, a5, -8 + +.Llt8: + st.d a6, a0, 0 + st.d a7, a2, -8 + + /* return */ + jr ra +SYM_FUNC_END(__rmemcpy_fast) diff --git a/arch/loongarch/lib/memset.S b/arch/loongarch/lib/memset.S new file mode 100644 index 0000000000000000000000000000000000000000..db8426647bf84945ba5c12e709de1625bb87ee4e --- /dev/null +++ b/arch/loongarch/lib/memset.S @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include + +.macro fill_to_64 r0 + bstrins.d \r0, \r0, 15, 8 + bstrins.d \r0, \r0, 31, 16 + bstrins.d \r0, \r0, 63, 32 +.endm + +SYM_FUNC_START_WEAK(memset) +SYM_FUNC_START_ALIAS(__memset) + /* + * Some CPUs support hardware unaligned access + */ + ALTERNATIVE "b __memset_generic", \ + "b __memset_fast", CPU_FEATURE_UAL +SYM_FUNC_END(memset) +SYM_FUNC_END_ALIAS(__memset) + +EXPORT_SYMBOL(memset) +EXPORT_SYMBOL(__memset) + +/* + * void *__memset_generic(void *s, int c, size_t n) + * + * a0: s + * a1: c + * a2: n + */ +SYM_FUNC_START(__memset_generic) + move a3, a0 + beqz a2, 2f + +1: st.b a1, a0, 0 + addi.d a0, a0, 1 + addi.d a2, a2, -1 + bgt a2, zero, 1b + +2: move a0, a3 + jr ra +SYM_FUNC_END(__memset_generic) + +/* + * void *__memset_fast(void *s, int c, size_t n) + * + * a0: s + * a1: c + * a2: n + */ +SYM_FUNC_START(__memset_fast) + /* fill a1 to 64 bits */ + fill_to_64 a1 + + sltui t0, a2, 9 + bnez t0, .Lsmall + + add.d a2, a0, a2 + st.d a1, a0, 0 + + /* align up address */ + addi.d a3, a0, 8 + bstrins.d a3, zero, 2, 0 + + addi.d a4, a2, -64 + bgeu a3, a4, .Llt64 + + /* set 64 bytes at a time */ +.Lloop64: + st.d a1, a3, 0 + st.d a1, a3, 8 + st.d a1, a3, 16 + st.d a1, a3, 24 + st.d a1, a3, 32 + st.d a1, a3, 40 + st.d a1, a3, 48 + st.d a1, a3, 56 + addi.d a3, a3, 64 + bltu a3, a4, .Lloop64 + + /* set the remaining bytes */ +.Llt64: + addi.d a4, a2, -32 + bgeu a3, a4, .Llt32 + st.d a1, a3, 0 + st.d a1, a3, 8 + st.d a1, a3, 16 + st.d a1, a3, 24 + addi.d a3, a3, 32 + +.Llt32: + addi.d a4, a2, -16 + bgeu a3, a4, .Llt16 + st.d a1, a3, 0 + st.d a1, a3, 8 + addi.d a3, a3, 16 + +.Llt16: + addi.d a4, a2, -8 + bgeu a3, a4, .Llt8 + st.d a1, a3, 0 + +.Llt8: + st.d a1, a2, -8 + + /* return */ + jr ra + + .align 4 +.Lsmall: + pcaddi t0, 4 + slli.d a2, a2, 4 + add.d t0, t0, a2 + jr t0 + + .align 4 +0: jr ra + + .align 4 +1: st.b a1, a0, 0 + jr ra + + .align 4 +2: st.h a1, a0, 0 + jr ra + + .align 4 +3: st.h a1, a0, 0 + st.b a1, a0, 2 + jr ra + + .align 4 +4: st.w a1, a0, 0 + jr ra + + .align 4 +5: st.w a1, a0, 0 + st.b a1, a0, 4 + jr ra + + .align 4 +6: st.w a1, a0, 0 + st.h a1, a0, 4 + jr ra + + .align 4 +7: st.w a1, a0, 0 + st.w a1, a0, 3 + jr ra + + .align 4 +8: st.d a1, a0, 0 + jr ra +SYM_FUNC_END(__memset_fast) diff --git a/arch/loongarch/lib/unaligned.S b/arch/loongarch/lib/unaligned.S new file mode 100644 index 0000000000000000000000000000000000000000..9177fd638f072c9b1cec989de31d9a13360e16cb --- /dev/null +++ b/arch/loongarch/lib/unaligned.S @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include + +#include +#include +#include +#include +#include +#include + +.L_fixup_handle_unaligned: + li.w a0, -EFAULT + jr ra + +/* + * unsigned long unaligned_read(void *addr, void *value, unsigned long n, bool sign) + * + * a0: addr + * a1: value + * a2: n + * a3: sign + */ +SYM_FUNC_START(unaligned_read) + beqz a2, 5f + + li.w t2, 0 + addi.d t0, a2, -1 + slli.d t1, t0, 3 + add.d a0, a0, t0 + + beqz a3, 2f +1: ld.b t3, a0, 0 + b 3f + +2: ld.bu t3, a0, 0 +3: sll.d t3, t3, t1 + or t2, t2, t3 + addi.d t1, t1, -8 + addi.d a0, a0, -1 + addi.d a2, a2, -1 + bgtz a2, 2b +4: st.d t2, a1, 0 + + move a0, a2 + jr ra + +5: li.w a0, -EFAULT + jr ra + + _asm_extable 1b, .L_fixup_handle_unaligned + _asm_extable 2b, .L_fixup_handle_unaligned + _asm_extable 4b, .L_fixup_handle_unaligned +SYM_FUNC_END(unaligned_read) + +/* + * unsigned long unaligned_write(void *addr, unsigned long value, unsigned long n) + * + * a0: addr + * a1: value + * a2: n + */ +SYM_FUNC_START(unaligned_write) + beqz a2, 3f + + li.w t0, 0 +1: srl.d t1, a1, t0 +2: st.b t1, a0, 0 + addi.d t0, t0, 8 + addi.d a2, a2, -1 + addi.d a0, a0, 1 + bgtz a2, 1b + + move a0, a2 + jr ra + +3: li.w a0, -EFAULT + jr ra + + _asm_extable 2b, .L_fixup_handle_unaligned +SYM_FUNC_END(unaligned_write) diff --git a/arch/loongarch/lib/xor_simd.c b/arch/loongarch/lib/xor_simd.c new file mode 100644 index 0000000000000000000000000000000000000000..84cd24b728c47968344ef2b771895e0e321262dd --- /dev/null +++ b/arch/loongarch/lib/xor_simd.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * LoongArch SIMD XOR operations + * + * Copyright (C) 2023 WANG Xuerui + */ + +#include "xor_simd.h" + +/* + * Process one cache line (64 bytes) per loop. This is assuming all future + * popular LoongArch cores are similar performance-characteristics-wise to the + * current models. + */ +#define LINE_WIDTH 64 + +#ifdef CONFIG_CPU_HAS_LSX + +#define LD(reg, base, offset) \ + "vld $vr" #reg ", %[" #base "], " #offset "\n\t" +#define ST(reg, base, offset) \ + "vst $vr" #reg ", %[" #base "], " #offset "\n\t" +#define XOR(dj, k) "vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t" + +#define LD_INOUT_LINE(base) \ + LD(0, base, 0) \ + LD(1, base, 16) \ + LD(2, base, 32) \ + LD(3, base, 48) + +#define LD_AND_XOR_LINE(base) \ + LD(4, base, 0) \ + LD(5, base, 16) \ + LD(6, base, 32) \ + LD(7, base, 48) \ + XOR(0, 4) \ + XOR(1, 5) \ + XOR(2, 6) \ + XOR(3, 7) + +#define ST_LINE(base) \ + ST(0, base, 0) \ + ST(1, base, 16) \ + ST(2, base, 32) \ + ST(3, base, 48) + +#define XOR_FUNC_NAME(nr) __xor_lsx_##nr +#include "xor_template.c" + +#undef LD +#undef ST +#undef XOR +#undef LD_INOUT_LINE +#undef LD_AND_XOR_LINE +#undef ST_LINE +#undef XOR_FUNC_NAME + +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX + +#define LD(reg, base, offset) \ + "xvld $xr" #reg ", %[" #base "], " #offset "\n\t" +#define ST(reg, base, offset) \ + "xvst $xr" #reg ", %[" #base "], " #offset "\n\t" +#define XOR(dj, k) "xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t" + +#define LD_INOUT_LINE(base) \ + LD(0, base, 0) \ + LD(1, base, 32) + +#define LD_AND_XOR_LINE(base) \ + LD(2, base, 0) \ + LD(3, base, 32) \ + XOR(0, 2) \ + XOR(1, 3) + +#define ST_LINE(base) \ + ST(0, base, 0) \ + ST(1, base, 32) + +#define XOR_FUNC_NAME(nr) __xor_lasx_##nr +#include "xor_template.c" + +#undef LD +#undef ST +#undef XOR +#undef LD_INOUT_LINE +#undef LD_AND_XOR_LINE +#undef ST_LINE +#undef XOR_FUNC_NAME + +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/arch/loongarch/lib/xor_simd.h b/arch/loongarch/lib/xor_simd.h new file mode 100644 index 0000000000000000000000000000000000000000..6555f84a7c97884e1de5dcaa7cd428c5bdc9a34e --- /dev/null +++ b/arch/loongarch/lib/xor_simd.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Simple interface to link xor_simd.c and xor_simd_glue.c + * + * Separating these files ensures that no SIMD instructions are run outside of + * the kfpu critical section. + */ + +#ifndef __LOONGARCH_LIB_XOR_SIMD_H +#define __LOONGARCH_LIB_XOR_SIMD_H + +#ifdef CONFIG_CPU_HAS_LSX +void __xor_lsx_2(unsigned long bytes, unsigned long *p1, + unsigned long *p2); +void __xor_lsx_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3); +void __xor_lsx_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4); +void __xor_lsx_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5); +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +void __xor_lasx_2(unsigned long bytes, unsigned long *p1, + unsigned long *p2); +void __xor_lasx_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3); +void __xor_lasx_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4); +void __xor_lasx_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5); +#endif /* CONFIG_CPU_HAS_LASX */ + +#endif /* __LOONGARCH_LIB_XOR_SIMD_H */ diff --git a/arch/loongarch/lib/xor_simd_glue.c b/arch/loongarch/lib/xor_simd_glue.c new file mode 100644 index 0000000000000000000000000000000000000000..f5b433ff7230b73fa94fd556ba19420474320faa --- /dev/null +++ b/arch/loongarch/lib/xor_simd_glue.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * LoongArch SIMD XOR operations + * + * Copyright (C) 2023 WANG Xuerui + */ + +#include +#include +#include +#include +#include "xor_simd.h" + +#define MAKE_XOR_GLUE_2(flavor) \ +void xor_##flavor##_2(unsigned long bytes, unsigned long *p1, \ + unsigned long *p2) \ +{ \ + kernel_fpu_begin(); \ + __xor_##flavor##_2(bytes, p1, p2); \ + kernel_fpu_end(); \ +} \ +EXPORT_SYMBOL_GPL(xor_##flavor##_2) + +#define MAKE_XOR_GLUE_3(flavor) \ +void xor_##flavor##_3(unsigned long bytes, unsigned long *p1, \ + unsigned long *p2, unsigned long *p3) \ +{ \ + kernel_fpu_begin(); \ + __xor_##flavor##_3(bytes, p1, p2, p3); \ + kernel_fpu_end(); \ +} \ +EXPORT_SYMBOL_GPL(xor_##flavor##_3) + +#define MAKE_XOR_GLUE_4(flavor) \ +void xor_##flavor##_4(unsigned long bytes, unsigned long *p1, \ + unsigned long *p2, unsigned long *p3, \ + unsigned long *p4) \ +{ \ + kernel_fpu_begin(); \ + __xor_##flavor##_4(bytes, p1, p2, p3, p4); \ + kernel_fpu_end(); \ +} \ +EXPORT_SYMBOL_GPL(xor_##flavor##_4) + +#define MAKE_XOR_GLUE_5(flavor) \ +void xor_##flavor##_5(unsigned long bytes, unsigned long *p1, \ + unsigned long *p2, unsigned long *p3, \ + unsigned long *p4, unsigned long *p5) \ +{ \ + kernel_fpu_begin(); \ + __xor_##flavor##_5(bytes, p1, p2, p3, p4, p5); \ + kernel_fpu_end(); \ +} \ +EXPORT_SYMBOL_GPL(xor_##flavor##_5) + +#define MAKE_XOR_GLUES(flavor) \ + MAKE_XOR_GLUE_2(flavor); \ + MAKE_XOR_GLUE_3(flavor); \ + MAKE_XOR_GLUE_4(flavor); \ + MAKE_XOR_GLUE_5(flavor) + +#ifdef CONFIG_CPU_HAS_LSX +MAKE_XOR_GLUES(lsx); +#endif + +#ifdef CONFIG_CPU_HAS_LASX +MAKE_XOR_GLUES(lasx); +#endif diff --git a/arch/loongarch/lib/xor_template.c b/arch/loongarch/lib/xor_template.c new file mode 100644 index 0000000000000000000000000000000000000000..1f828860a25cdf4663a4499000c05044c708d502 --- /dev/null +++ b/arch/loongarch/lib/xor_template.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 WANG Xuerui + * + * Template for XOR operations, instantiated in xor_simd.c. + * + * Expected preprocessor definitions: + * + * - LINE_WIDTH + * - XOR_FUNC_NAME(nr) + * - LD_INOUT_LINE(buf) + * - LD_AND_XOR_LINE(buf) + * - ST_LINE(buf) + */ + +void XOR_FUNC_NAME(2)(unsigned long bytes, + unsigned long *v1, unsigned long *v2) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(3)(unsigned long bytes, + unsigned long *v1, unsigned long *v2, + unsigned long *v3) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(4)(unsigned long bytes, + unsigned long *v1, unsigned long *v2, + unsigned long *v3, unsigned long *v4) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + LD_AND_XOR_LINE(v4) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4) + : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + v4 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} + +void XOR_FUNC_NAME(5)(unsigned long bytes, + unsigned long *v1, unsigned long *v2, + unsigned long *v3, unsigned long *v4, + unsigned long *v5) +{ + unsigned long lines = bytes / LINE_WIDTH; + + do { + __asm__ __volatile__ ( + LD_INOUT_LINE(v1) + LD_AND_XOR_LINE(v2) + LD_AND_XOR_LINE(v3) + LD_AND_XOR_LINE(v4) + LD_AND_XOR_LINE(v5) + ST_LINE(v1) + : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4), + [v5] "r"(v5) : "memory" + ); + + v1 += LINE_WIDTH / sizeof(unsigned long); + v2 += LINE_WIDTH / sizeof(unsigned long); + v3 += LINE_WIDTH / sizeof(unsigned long); + v4 += LINE_WIDTH / sizeof(unsigned long); + v5 += LINE_WIDTH / sizeof(unsigned long); + } while (--lines > 0); +} diff --git a/arch/loongarch/loongson64/Makefile b/arch/loongarch/loongson64/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..3e54460d8d001b9e88736a26e8b79411138c60f6 --- /dev/null +++ b/arch/loongarch/loongson64/Makefile @@ -0,0 +1,9 @@ +# +# All Loongson based systems +# + +obj-y += setup.o init.o env.o reset.o irq.o mem.o dma.o rtc.o + +obj-$(CONFIG_SMP) += smp.o + +obj-$(CONFIG_NUMA) += numa.o diff --git a/arch/loongarch/loongson64/Platform b/arch/loongarch/loongson64/Platform new file mode 100644 index 0000000000000000000000000000000000000000..15a033be4699d2018eb9c1b23911fd3676f7c857 --- /dev/null +++ b/arch/loongarch/loongson64/Platform @@ -0,0 +1,13 @@ +# +# Loongson Processors' Support +# + +cflags-$(CONFIG_CPU_LOONGSON64) += $(call as-option,-Wa$(comma)-mno-fix-loongson3-llsc,) + +# +# Loongson Machines Support +# + +platform-$(CONFIG_MACH_LOONGSON64) += loongson64/ +cflags-$(CONFIG_MACH_LOONGSON64) += -I$(srctree)/arch/loongarch/include/asm/mach-loongson64 +load-$(CONFIG_MACH_LOONGSON64) += 0x9000000000200000 diff --git a/arch/loongarch/loongson64/dma.c b/arch/loongarch/loongson64/dma.c new file mode 100644 index 0000000000000000000000000000000000000000..0f9040535cbb797f9a2d392d10e29d47901738f2 --- /dev/null +++ b/arch/loongarch/loongson64/dma.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + */ +#include +#include + +void acpi_arch_dma_setup(struct device *dev) +{ + int ret; + u64 mask, end = 0; + const struct bus_dma_region *map = NULL; + + ret = acpi_dma_get_range(dev, &map); + if (!ret && map) { + const struct bus_dma_region *r = map; + + for (end = 0; r->size; r++) { + if (r->dma_start + r->size - 1 > end) + end = r->dma_start + r->size - 1; + } + + mask = DMA_BIT_MASK(ilog2(end) + 1); + dev->bus_dma_limit = end; + dev->dma_range_map = map; + dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); + *dev->dma_mask = min(*dev->dma_mask, mask); + } + +} diff --git a/arch/loongarch/loongson64/env.c b/arch/loongarch/loongson64/env.c new file mode 100644 index 0000000000000000000000000000000000000000..9383bb209d417ea4799108212386aa3f780091c7 --- /dev/null +++ b/arch/loongarch/loongson64/env.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include + +u64 efi_system_table; +unsigned long long smp_group[MAX_PACKAGES]; +struct loongson_system_configuration loongson_sysconf; +EXPORT_SYMBOL(loongson_sysconf); + +static void __init register_addrs_set(u64 *registers, const u64 addr, int num) +{ + u64 i; + + for (i = 0; i < num; i++) { + *registers = (i << 44) | addr; + registers++; + } +} + +void __init init_environ(void) +{ + int efi_boot = fw_arg0; + char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); + + if (efi_boot) + set_bit(EFI_BOOT, &efi.flags); + else + clear_bit(EFI_BOOT, &efi.flags); + + strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); + early_memunmap(cmdline, COMMAND_LINE_SIZE); + + efi_system_table = fw_arg2; + register_addrs_set(smp_group, TO_UNCACHE(0x1fe01000), 16); +} + +static int __init init_cpu_fullname(void) +{ + int cpu; + + if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { + for (cpu = 0; cpu < NR_CPUS; cpu++) + __cpu_full_name[cpu] = loongson_sysconf.cpuname; + } + return 0; +} +arch_initcall(init_cpu_fullname); + +static ssize_t boardinfo_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, + "BIOS Information\n" + "Vendor\t\t\t: %s\n" + "Version\t\t\t: %s\n" + "ROM Size\t\t: %d KB\n" + "Release Date\t\t: %s\n\n" + "Board Information\n" + "Manufacturer\t\t: %s\n" + "Board Name\t\t: %s\n" + "Family\t\t\t: LOONGSON64\n\n", + b_info.bios_vendor, b_info.bios_version, + b_info.bios_size, b_info.bios_release_date, + b_info.board_vendor, b_info.board_name); +} + +static struct kobj_attribute boardinfo_attr = __ATTR(boardinfo, 0444, + boardinfo_show, NULL); + +static int __init boardinfo_init(void) +{ + struct kobject *loongson_kobj; + + loongson_kobj = kobject_create_and_add("loongson", firmware_kobj); + + return sysfs_create_file(loongson_kobj, &boardinfo_attr.attr); +} +late_initcall(boardinfo_init); diff --git a/arch/loongarch/loongson64/init.c b/arch/loongarch/loongson64/init.c new file mode 100644 index 0000000000000000000000000000000000000000..d5ffdc82b3c2aed5ddf49aaa393fcf064012942f --- /dev/null +++ b/arch/loongarch/loongson64/init.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define SMBIOS_BIOSSIZE_OFFSET 0x09 +#define SMBIOS_BIOSEXTERN_OFFSET 0x13 +#define SMBIOS_FREQLOW_OFFSET 0x16 +#define SMBIOS_FREQHIGH_OFFSET 0x17 +#define SMBIOS_FREQLOW_MASK 0xFF +#define SMBIOS_CORE_PACKAGE_OFFSET 0x23 +#define LOONGSON_EFI_ENABLE (1 << 3) + +struct loongson_board_info b_info; +static const char dmi_empty_string[] = " "; +extern void __init arch_reserve_vmcore(void); +extern void __init arch_parse_crashkernel(void); + +static const char *dmi_string_parse(const struct dmi_header *dm, u8 s) +{ + const u8 *bp = ((u8 *) dm) + dm->length; + + if (s) { + s--; + while (s > 0 && *bp) { + bp += strlen(bp) + 1; + s--; + } + + if (*bp != 0) { + size_t len = strlen(bp)+1; + size_t cmp_len = len > 8 ? 8 : len; + + if (!memcmp(bp, dmi_empty_string, cmp_len)) + return dmi_empty_string; + + return bp; + } + } + + return ""; + +} + +static void __init parse_cpu_table(const struct dmi_header *dm) +{ + long freq_temp = 0; + char *dmi_data = (char *)dm; + + freq_temp = ((*(dmi_data + SMBIOS_FREQHIGH_OFFSET) << 8) + \ + ((*(dmi_data + SMBIOS_FREQLOW_OFFSET)) & SMBIOS_FREQLOW_MASK)); + cpu_clock_freq = freq_temp * 1000000; + + loongson_sysconf.cpuname = (void *)dmi_string_parse(dm, dmi_data[16]); + loongson_sysconf.cores_per_package = *(dmi_data + SMBIOS_CORE_PACKAGE_OFFSET); + + pr_info("CpuClock = %llu\n", cpu_clock_freq); + +} + +static void __init parse_bios_table(const struct dmi_header *dm) +{ + char *dmi_data = (char *)dm; + + b_info.bios_size = (*(dmi_data + SMBIOS_BIOSSIZE_OFFSET) + 1) << 6; +} + +static void __init find_tokens(const struct dmi_header *dm, void *dummy) +{ + switch (dm->type) { + case 0x0: /* Extern BIOS */ + parse_bios_table(dm); + break; + case 0x4: /* Calling interface */ + parse_cpu_table(dm); + break; + } +} + +static void __init smbios_parse(void) +{ + b_info.bios_vendor = (void *)dmi_get_system_info(DMI_BIOS_VENDOR); + b_info.bios_version = (void *)dmi_get_system_info(DMI_BIOS_VERSION); + b_info.bios_release_date = (void *)dmi_get_system_info(DMI_BIOS_DATE); + b_info.board_vendor = (void *)dmi_get_system_info(DMI_BOARD_VENDOR); + b_info.board_name = (void *)dmi_get_system_info(DMI_BOARD_NAME); + dmi_walk(find_tokens, NULL); +} + +void __init early_init(void) +{ + init_environ(); + efi_init(); + memblock_init(); +} + +void __init platform_init(void) +{ + arch_reserve_vmcore(); + arch_parse_crashkernel(); + +#ifdef CONFIG_ACPI_TABLE_UPGRADE + acpi_table_upgrade(); +#endif +#ifdef CONFIG_ACPI + acpi_gbl_use_default_register_widths = false; + acpi_boot_table_init(); +#endif + +#ifdef CONFIG_NUMA + init_numa_memory(); +#endif + dmi_setup(); + smbios_parse(); + pr_info("The BIOS Version: %s\n",b_info.bios_version); + + efi_runtime_init(); + + register_smp_ops(&loongson3_smp_ops); +} diff --git a/arch/loongarch/loongson64/irq.c b/arch/loongarch/loongson64/irq.c new file mode 100644 index 0000000000000000000000000000000000000000..ac1294652f3618a4f35f426a0eb5e47517e67725 --- /dev/null +++ b/arch/loongarch/loongson64/irq.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct acpi_madt_lio_pic *acpi_liointc; +struct acpi_madt_eio_pic *acpi_eiointc[MAX_IO_PICS]; + +struct acpi_madt_ht_pic *acpi_htintc; +struct acpi_madt_lpc_pic *acpi_pchlpc; +struct acpi_madt_msi_pic *acpi_pchmsi[MAX_IO_PICS]; +struct acpi_madt_bio_pic *acpi_pchpic[MAX_IO_PICS]; + +struct irq_domain *cpu_domain; +struct irq_domain *liointc_domain; +struct irq_domain *pch_lpc_domain; +struct irq_domain *pch_msi_domain[MAX_IO_PICS]; +struct irq_domain *pch_pic_domain[MAX_IO_PICS]; + +int find_pch_pic(u32 gsi) +{ + int i, start, end; + + /* Find the PCH_PIC that manages this GSI. */ + for(i = 0; i < loongson_sysconf.nr_io_pics; i++) { + struct acpi_madt_bio_pic *irq_cfg = acpi_pchpic[i]; + + start = irq_cfg->gsi_base; + end = irq_cfg->gsi_base + irq_cfg->size; + if (gsi >= start && gsi < end) + return i; + } + + printk(KERN_ERR "ERROR: Unable to locate PCH_PIC for GSI %d\n", gsi); + return -1; +} + +void __init setup_IRQ(void) +{ + int i; + struct irq_domain *parent_domain; + + if (!acpi_eiointc[0]) + cpu_data[0].options &= ~LOONGARCH_CPU_EXTIOI; + + cpu_domain = loongarch_cpu_irq_init(); + liointc_domain = liointc_acpi_init(cpu_domain, acpi_liointc); + + if (cpu_has_extioi) { + pr_info("Using EIOINTC interrupt mode\n"); + for(i = 0; i < loongson_sysconf.nr_io_pics; i++) { + parent_domain = eiointc_acpi_init(cpu_domain, acpi_eiointc[i]); + pch_pic_domain[i] = pch_pic_acpi_init(parent_domain, acpi_pchpic[i]); + pch_msi_domain[i] = pch_msi_acpi_init(parent_domain, acpi_pchmsi[i]); + } + } else { + pr_info("Using HTVECINTC interrupt mode\n"); + parent_domain = htvec_acpi_init(liointc_domain, acpi_htintc); + pch_pic_domain[0] = pch_pic_acpi_init(parent_domain, acpi_pchpic[0]); + pch_msi_domain[0] = pch_msi_acpi_init(parent_domain, acpi_pchmsi[0]); + } + + irq_set_default_host(pch_pic_domain[0]); + pch_lpc_domain = pch_lpc_acpi_init(pch_pic_domain[0], acpi_pchlpc); +} + +void __init arch_init_irq(void) +{ + int r, ipi_irq; + static int ipi_dummy_dev; + + clear_csr_ecfg(ECFG0_IM); + clear_csr_estat(ESTATF_IP); + + setup_IRQ(); +#ifdef CONFIG_SMP + ipi_irq = get_ipi_irq(); + irq_set_percpu_devid(ipi_irq); + r = request_percpu_irq(ipi_irq, loongson3_ipi_interrupt, "IPI", &ipi_dummy_dev); + if (r < 0) + panic("IPI IRQ request failed\n"); +#endif + + set_csr_ecfg(ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | ECFGF_IPI | ECFGF_PMC); +} diff --git a/arch/loongarch/loongson64/mem.c b/arch/loongarch/loongson64/mem.c new file mode 100644 index 0000000000000000000000000000000000000000..44bd3c184dd6c00e15e1ed5cab0096d5d51c047f --- /dev/null +++ b/arch/loongarch/loongson64/mem.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include +#include + +#include +#include + +#include + +void __init memblock_init(void) +{ + u32 mem_type; + u64 mem_start, mem_end, mem_size; + efi_memory_desc_t *md; + + /* Parse memory information */ + for_each_efi_memory_desc(md) { + mem_type = md->type; + mem_start = md->phys_addr; + mem_size = md->num_pages << EFI_PAGE_SHIFT; + mem_end = mem_start + mem_size; + + switch (mem_type) { + case EFI_LOADER_CODE: + case EFI_LOADER_DATA: + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + case EFI_PERSISTENT_MEMORY: + case EFI_CONVENTIONAL_MEMORY: + memblock_add(mem_start, mem_size); + if (max_low_pfn < (mem_end >> PAGE_SHIFT)) + max_low_pfn = mem_end >> PAGE_SHIFT; + break; + case EFI_PAL_CODE: + case EFI_UNUSABLE_MEMORY: + case EFI_ACPI_RECLAIM_MEMORY: + memblock_add(mem_start, mem_size); + fallthrough; + case EFI_RESERVED_TYPE: + case EFI_RUNTIME_SERVICES_CODE: + case EFI_RUNTIME_SERVICES_DATA: + case EFI_MEMORY_MAPPED_IO: + case EFI_MEMORY_MAPPED_IO_PORT_SPACE: + memblock_reserve(mem_start, mem_size); + break; + } + } + + memblock_set_current_limit(PFN_PHYS(max_low_pfn)); + + /* Reserve the first 2MB */ + memblock_reserve(PHYS_OFFSET, 0x200000); + + /* Reserve the kernel text/data/bss */ + memblock_reserve(__pa_symbol(&_text), + __pa_symbol(&_end) - __pa_symbol(&_text)); + + memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); + memblock_set_node(0, PHYS_ADDR_MAX, &memblock.reserved, 0); +} diff --git a/arch/loongarch/loongson64/numa.c b/arch/loongarch/loongson64/numa.c new file mode 100644 index 0000000000000000000000000000000000000000..5abf1925ed095a9c85437c3e15d6b7df3e6ae1cb --- /dev/null +++ b/arch/loongarch/loongson64/numa.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Xiang Gao + * Huacai Chen + * + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int numa_off; +struct pglist_data *node_data[MAX_NUMNODES]; +unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES]; + +EXPORT_SYMBOL(node_data); +EXPORT_SYMBOL(node_distances); + +static struct numa_meminfo numa_meminfo; +cpumask_t cpus_on_node[MAX_NUMNODES]; +cpumask_t phys_cpus_on_node[MAX_NUMNODES]; +EXPORT_SYMBOL(cpus_on_node); + +/* + * apicid, cpu, node mappings + */ +s16 __cpuid_to_node[CONFIG_NR_CPUS] = { + [0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE +}; +EXPORT_SYMBOL(__cpuid_to_node); + +nodemask_t numa_nodes_parsed __initdata; + +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + if (early_cpu_to_node(from) == early_cpu_to_node(to)) + return LOCAL_DISTANCE; + else + return REMOTE_DISTANCE; +} + +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, + size_t align) +{ + return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_ACCESSIBLE, early_cpu_to_node(cpu)); +} + +static void __init pcpu_fc_free(void *ptr, size_t size) +{ + memblock_free_early(__pa(ptr), size); +} + +static void __init pcpu_populate_pte(unsigned long addr) +{ + populate_kernel_pte(addr); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc = -EINVAL; + + if (pcpu_chosen_fc == PCPU_FC_AUTO) { + if (nr_node_ids >= 8) + pcpu_chosen_fc = PCPU_FC_PAGE; + else + pcpu_chosen_fc = PCPU_FC_EMBED; + } + + /* + * Always reserve area for module percpu variables. That's + * what the legacy allocator did. + */ + if (pcpu_chosen_fc != PCPU_FC_PAGE) { + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, PMD_SIZE, + pcpu_cpu_distance, + pcpu_fc_alloc, pcpu_fc_free); + if (rc < 0) + pr_warn("%s allocator failed (%d), falling back to page size\n", + pcpu_fc_names[pcpu_chosen_fc], rc); + } + if (rc < 0) + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, + pcpu_fc_alloc, pcpu_fc_free, + pcpu_populate_pte); + if (rc < 0) + panic("cannot initialize percpu area (err=%d)", rc); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; +} +#endif + +/* + * Get nodeid by logical cpu number. + * __cpuid_to_node maps phyical cpu id to node, so we + * should use cpu_logical_map(cpu) to index it. + * + * This routine is only used in early phase during + * booting, after setup_per_cpu_areas calling and numa_node + * initialization, cpu_to_node will be used instead. + * */ +int early_cpu_to_node(int cpu) +{ + int physid = cpu_logical_map(cpu); + + if (physid < 0) + return NUMA_NO_NODE; + + return __cpuid_to_node[physid]; +} + +void __init early_numa_add_cpu(int cpuid, s16 node) +{ + int cpu = __cpu_number_map[cpuid]; + + if (cpu < 0) + return; + + cpumask_set_cpu(cpu, &cpus_on_node[node]); + cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]); +} + +void numa_add_cpu(unsigned int cpu) +{ + int nid = cpu_to_node(cpu); + cpumask_set_cpu(cpu, &cpus_on_node[nid]); +} + +void numa_remove_cpu(unsigned int cpu) +{ + int nid = cpu_to_node(cpu); + cpumask_clear_cpu(cpu, &cpus_on_node[nid]); +} + +static int __init numa_add_memblk_to(int nid, u64 start, u64 end, + struct numa_meminfo *mi) +{ + /* ignore zero length blks */ + if (start == end) + return 0; + + /* whine about and ignore invalid blks */ + if (start > end || nid < 0 || nid >= MAX_NUMNODES) { + pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", + nid, start, end - 1); + return 0; + } + + if (mi->nr_blks >= NR_NODE_MEMBLKS) { + pr_err("NUMA: too many memblk ranges\n"); + return -EINVAL; + } + + mi->blk[mi->nr_blks].start = PFN_ALIGN(start); + mi->blk[mi->nr_blks].end = PFN_ALIGN(end - PAGE_SIZE + 1); + mi->blk[mi->nr_blks].nid = nid; + mi->nr_blks++; + return 0; +} + +/** + * numa_add_memblk - Add one numa_memblk to numa_meminfo + * @nid: NUMA node ID of the new memblk + * @start: Start address of the new memblk + * @end: End address of the new memblk + * + * Add a new memblk to the default numa_meminfo. + * + * RETURNS: + * 0 on success, -errno on failure. + */ +int __init numa_add_memblk(int nid, u64 start, u64 end) +{ + return numa_add_memblk_to(nid, start, end, &numa_meminfo); +} + +static void __init alloc_node_data(int nid) +{ + void *nd; + unsigned long nd_pa; + size_t nd_sz = roundup(sizeof(pg_data_t), PAGE_SIZE); + + nd_pa = memblock_phys_alloc_try_nid(nd_sz, SMP_CACHE_BYTES, nid); + if (!nd_pa) { + pr_err("Cannot find %zu Byte for node_data (initial node: %d)\n", nd_sz, nid); + return; + } + + nd = __va(nd_pa); + + node_data[nid] = nd; + memset(nd, 0, sizeof(pg_data_t)); +} + +static void __init node_mem_init(unsigned int node) +{ + unsigned long start_pfn, end_pfn; + unsigned long node_addrspace_offset; + + node_addrspace_offset = nid_to_addrbase(node); + pr_info("Node%d's addrspace_offset is 0x%lx\n", + node, node_addrspace_offset); + + get_pfn_range_for_nid(node, &start_pfn, &end_pfn); + pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n", + node, start_pfn, end_pfn); + + alloc_node_data(node); +} + +#ifdef CONFIG_ACPI_NUMA + +/* + * Sanity check to catch more bad NUMA configurations (they are amazingly + * common). Make sure the nodes cover all memory. + */ +static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi) +{ + u64 numaram, biosram; + int i; + + numaram = 0; + for (i = 0; i < mi->nr_blks; i++) { + u64 s = mi->blk[i].start >> PAGE_SHIFT; + u64 e = mi->blk[i].end >> PAGE_SHIFT; + numaram += e - s; + numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e); + if ((s64)numaram < 0) + numaram = 0; + } + max_pfn = max_low_pfn; + biosram = max_pfn - absent_pages_in_range(0, max_pfn); + + BUG_ON((s64)(biosram - numaram) >= (1 << (20 - PAGE_SHIFT))); + return true; +} + +static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type) +{ + static unsigned long num_physpages; + + num_physpages += (size >> PAGE_SHIFT); + pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", + node, type, start, size); + pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n", + start >> PAGE_SHIFT, (start + size) >> PAGE_SHIFT, num_physpages); + memblock_set_node(start, size, &memblock.memory, node); +} + +/* + * add_numamem_region + * + * Add a uasable memory region described by BIOS. The + * routine gets each intersection between BIOS's region + * and node's region, and adds them into node's memblock + * pool. + * + */ +static void __init add_numamem_region(u64 start, u64 end, u32 type) +{ + u32 i; + u64 ofs = start; + + if (start >= end) { + pr_debug("Invalid region: %016llx-%016llx\n", start, end); + return; + } + + for (i = 0; i < numa_meminfo.nr_blks; i++) { + struct numa_memblk *mb = &numa_meminfo.blk[i]; + + if (ofs > mb->end) + continue; + + if (end > mb->end) { + add_node_intersection(mb->nid, ofs, mb->end - ofs, type); + ofs = mb->end; + } else { + add_node_intersection(mb->nid, ofs, end - ofs, type); + break; + } + } +} + +static void __init init_node_memblock(void) +{ + u32 mem_type; + u64 mem_end, mem_start, mem_size; + efi_memory_desc_t *md; + + /* Parse memory information and activate */ + for_each_efi_memory_desc(md) { + mem_type = md->type; + mem_start = md->phys_addr; + mem_size = md->num_pages << EFI_PAGE_SHIFT; + mem_end = mem_start + mem_size; + + switch (mem_type) { + case EFI_LOADER_CODE: + case EFI_LOADER_DATA: + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + case EFI_PERSISTENT_MEMORY: + case EFI_CONVENTIONAL_MEMORY: + add_numamem_region(mem_start, mem_end, mem_type); + break; + case EFI_PAL_CODE: + case EFI_UNUSABLE_MEMORY: + case EFI_ACPI_RECLAIM_MEMORY: + add_numamem_region(mem_start, mem_end, mem_type); + fallthrough; + case EFI_RESERVED_TYPE: + case EFI_RUNTIME_SERVICES_CODE: + case EFI_RUNTIME_SERVICES_DATA: + case EFI_MEMORY_MAPPED_IO: + case EFI_MEMORY_MAPPED_IO_PORT_SPACE: + pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n", + mem_type, mem_start, mem_size); + break; + } + } +} + +static void __init numa_default_distance(void) +{ + int row, col; + + for (row = 0; row < MAX_NUMNODES; row++) + for (col = 0; col < MAX_NUMNODES; col++) { + if (col == row) + node_distances[row][col] = LOCAL_DISTANCE; + else + /* We assume that one node per package here! + * + * A SLIT should be used for multiple nodes + * per package to override default setting. + */ + node_distances[row][col] = REMOTE_DISTANCE; + } +} + +int __init init_numa_memory(void) +{ + int i; + int ret; + int node; + + for (i = 0; i < NR_CPUS; i++) + set_cpuid_to_node(i, NUMA_NO_NODE); + + numa_default_distance(); + nodes_clear(numa_nodes_parsed); + nodes_clear(node_possible_map); + nodes_clear(node_online_map); + memset(&numa_meminfo, 0, sizeof(numa_meminfo)); + + /* Parse SRAT and SLIT if provided by firmware. */ + ret = acpi_numa_init(); + if (ret < 0) + return ret; + + node_possible_map = numa_nodes_parsed; + if (WARN_ON(nodes_empty(node_possible_map))) + return -EINVAL; + + init_node_memblock(); + if (numa_meminfo_cover_memory(&numa_meminfo) == false) + return -EINVAL; + + for_each_node_mask(node, node_possible_map) { + node_mem_init(node); + node_set_online(node); + } + max_low_pfn = PHYS_PFN(memblock_end_of_DRAM()); + + setup_nr_node_ids(); + loongson_sysconf.nr_nodes = nr_node_ids; + loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]); + + return 0; +} + +#endif + +void __init paging_init(void) +{ + unsigned int node; + unsigned long zones_size[MAX_NR_ZONES] = {0, }; + + for_each_online_node(node) { + unsigned long start_pfn, end_pfn; + + get_pfn_range_for_nid(node, &start_pfn, &end_pfn); + + if (end_pfn > max_low_pfn) + max_low_pfn = end_pfn; + } +#ifdef CONFIG_ZONE_DMA32 + zones_size[ZONE_DMA32] = MAX_DMA32_PFN; +#endif + zones_size[ZONE_NORMAL] = max_low_pfn; + free_area_init(zones_size); +} + +void __init mem_init(void) +{ + high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); + memblock_free_all(); + mem_init_print_info(NULL); +} + +int pcibus_to_node(struct pci_bus *bus) +{ + return dev_to_node(&bus->dev); +} +EXPORT_SYMBOL(pcibus_to_node); diff --git a/arch/loongarch/loongson64/reset.c b/arch/loongarch/loongson64/reset.c new file mode 100644 index 0000000000000000000000000000000000000000..8c4085f81618e7aa17adabf9288a99fcabba3fc1 --- /dev/null +++ b/arch/loongarch/loongson64/reset.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen, chenhuacai@loongson.cn + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void loongson_restart(void) +{ +#ifdef CONFIG_EFI + if (efi_capsule_pending(NULL)) + efi_reboot(REBOOT_WARM, NULL); + else + efi_reboot(REBOOT_COLD, NULL); +#endif + if (!acpi_disabled) + acpi_reboot(); + + while (1) { + __arch_cpu_idle(); + } +} + +static void loongson_poweroff(void) +{ +#ifdef CONFIG_EFI + efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); +#endif + while (1) { + __arch_cpu_idle(); + } +} + +static int __init loongarch_reboot_setup(void) +{ + pm_restart = loongson_restart; + pm_power_off = loongson_poweroff; + + return 0; +} + +arch_initcall(loongarch_reboot_setup); diff --git a/arch/loongarch/loongson64/rtc.c b/arch/loongarch/loongson64/rtc.c new file mode 100644 index 0000000000000000000000000000000000000000..2472f3a70f52331b9e957a31fd1850897cb1bd9c --- /dev/null +++ b/arch/loongarch/loongson64/rtc.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ + +#include +#include +#include +#include + +#define RTC_TOYREAD0 0x2C +#define RTC_YEAR 0x30 + +unsigned long loongson_get_rtc_time(void) +{ + unsigned int year, mon, day, hour, min, sec; + unsigned int value; + + value = ls7a_readl(LS7A_RTC_REG_BASE + RTC_TOYREAD0); + sec = (value >> 4) & 0x3f; + min = (value >> 10) & 0x3f; + hour = (value >> 16) & 0x1f; + day = (value >> 21) & 0x1f; + mon = (value >> 26) & 0x3f; + year = ls7a_readl(LS7A_RTC_REG_BASE + RTC_YEAR); + + year = 1900 + year; + + return mktime64(year, mon, day, hour, min, sec); +} + +void read_persistent_clock64(struct timespec64 *ts) +{ + ts->tv_sec = loongson_get_rtc_time(); + ts->tv_nsec = 0; +} diff --git a/arch/loongarch/loongson64/setup.c b/arch/loongarch/loongson64/setup.c new file mode 100644 index 0000000000000000000000000000000000000000..a37b699837df5149cf0e5912f66cde7befdba5bf --- /dev/null +++ b/arch/loongarch/loongson64/setup.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include +#include +#include +#include +#include + +#ifdef CONFIG_VT +#include +#include +#include +#endif + +#include + +const char *get_system_type(void) +{ + return "generic-loongson-machine"; +} + +void __init plat_mem_setup(void) +{ +} + +static int __init register_gop_device(void) +{ + void *pd; + if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI) + return 0; + pd = platform_device_register_data(NULL, "efi-framebuffer", 0, + &screen_info, sizeof(screen_info)); + return PTR_ERR_OR_ZERO(pd); +} +subsys_initcall(register_gop_device); + +#define NR_CELLS 6 + +void __init device_tree_init(void) +{ + if (!initial_boot_params) + return; + + if (early_init_dt_verify(initial_boot_params)) + unflatten_and_copy_device_tree(); +} diff --git a/arch/loongarch/loongson64/smp.c b/arch/loongarch/loongson64/smp.c new file mode 100644 index 0000000000000000000000000000000000000000..9f7df4236af3f525e68e1a964f3ea19184ad30d3 --- /dev/null +++ b/arch/loongarch/loongson64/smp.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct secondary_data cpuboot_data; + +static DEFINE_PER_CPU(int, cpu_state); +DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL(irq_stat); + +#define MAX_CPUS 64 + +#define STATUS 0x00 +#define EN 0x04 +#define SET 0x08 +#define CLEAR 0x0c +#define MBUF 0x20 + +extern unsigned long long smp_group[MAX_PACKAGES]; +static u32 core_offsets[4] = {0x000, 0x100, 0x200, 0x300}; + +static volatile void *ipi_set_regs[MAX_CPUS]; +static volatile void *ipi_clear_regs[MAX_CPUS]; +static volatile void *ipi_status_regs[MAX_CPUS]; +static volatile void *ipi_en_regs[MAX_CPUS]; +static volatile void *ipi_mailbox_buf[MAX_CPUS]; + +static u32 (*ipi_read_clear)(int cpu); +static void (*ipi_write_action)(int cpu, u32 action); + +enum ipi_msg_type { + IPI_RESCHEDULE, + IPI_CALL_FUNCTION, +}; + +static const char *ipi_types[NR_IPI] __tracepoint_string = { + [IPI_RESCHEDULE] = "Rescheduling interrupts", + [IPI_CALL_FUNCTION] = "Function call interrupts", +}; + +void show_ipi_list(struct seq_file *p, int prec) +{ + unsigned int cpu, i; + + for (i = 0; i < NR_IPI; i++) { + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); + for_each_online_cpu(cpu) + seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]); + seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); + } +} + +/* Send mail buffer via Mail_Send */ +static void csr_mail_send(uint64_t data, int cpu, int mailbox) +{ + uint64_t val; + + /* Send high 32 bits */ + val = IOCSR_MBUF_SEND_BLOCKING; + val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); + val |= (data & IOCSR_MBUF_SEND_H32_MASK); + iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); + + /* Send low 32 bits */ + val = IOCSR_MBUF_SEND_BLOCKING; + val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); + val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); + iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); +}; + +static u32 csr_ipi_read_clear(int cpu) +{ + u32 action; + + /* Load the ipi register to figure out what we're supposed to do */ + action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); + /* Clear the ipi register to clear the interrupt */ + iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); + + return action; +} + +static void csr_ipi_write_action(int cpu, u32 action) +{ + unsigned int irq = 0; + + while ((irq = ffs(action))) { + uint32_t val = IOCSR_IPI_SEND_BLOCKING; + val |= (irq - 1); + val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); + iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); + action &= ~BIT(irq - 1); + } +} + +static u32 legacy_ipi_read_clear(int cpu) +{ + u32 action; + + /* Load the ipi register to figure out what we're supposed to do */ + action = xconf_readl(ipi_status_regs[cpu]); + /* Clear the ipi register to clear the interrupt */ + xconf_writel(action, ipi_clear_regs[cpu]); + + return action; +} + +static void legacy_ipi_write_action(int cpu, u32 action) +{ + xconf_writel((u32)action, ipi_set_regs[cpu]); +} + +static void ipi_method_init(void) +{ + if (cpu_has_csripi) { + ipi_read_clear = csr_ipi_read_clear; + ipi_write_action = csr_ipi_write_action; + } else { + ipi_read_clear = legacy_ipi_read_clear; + ipi_write_action = legacy_ipi_write_action; + } +} + +static void ipi_regaddrs_init(void) +{ + int i, node, core; + + for (i = 0; i< MAX_CPUS; i++) { + node = i / 4; + core = i % 4; + ipi_set_regs[i] = (void *) + (smp_group[node] + core_offsets[core] + SET); + ipi_clear_regs[i] = (void *) + (smp_group[node] + core_offsets[core] + CLEAR); + ipi_status_regs[i] = (void *) + (smp_group[node] + core_offsets[core] + STATUS); + ipi_en_regs[i] = (void *) + (smp_group[node] + core_offsets[core] + EN); + ipi_mailbox_buf[i] = (void *) + (smp_group[node] + core_offsets[core] + MBUF); + } +} + +/* + * Simple enough, just poke the appropriate ipi register + */ +static void loongson3_send_ipi_single(int cpu, unsigned int action) +{ + ipi_write_action(cpu_logical_map(cpu), (u32)action); +} + +static void +loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) +{ + unsigned int i; + + for_each_cpu(i, mask) + ipi_write_action(cpu_logical_map(i), (u32)action); +} + +irqreturn_t loongson3_ipi_interrupt(int irq, void *dev) +{ + unsigned int action; + unsigned int cpu = smp_processor_id(); + + action = ipi_read_clear(cpu_logical_map(cpu)); + + wbflush(); + + if (action & SMP_RESCHEDULE) { + scheduler_ipi(); + per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; + } + + if (action & SMP_CALL_FUNCTION) { + generic_smp_call_function_interrupt(); + per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; + } + + return IRQ_HANDLED; +} + +/* + * SMP init and finish on secondary CPUs + */ +static void loongson3_init_secondary(void) +{ + unsigned int cpu = smp_processor_id(); + unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | + ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER; + + change_csr_ecfg(ECFG0_IM, imask); + + if (cpu_has_csripi) + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); + else + xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(cpu)]); + +#ifdef CONFIG_NUMA + numa_add_cpu(cpu); +#endif + per_cpu(cpu_state, cpu) = CPU_ONLINE; + cpu_data[cpu].package = + cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; + cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : + cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; +} + +static void loongson3_smp_finish(void) +{ + int cpu = smp_processor_id(); + + local_irq_enable(); + + if (cpu_has_csripi) + iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); + else + xconf_writeq(0, ipi_mailbox_buf[cpu_logical_map(cpu)]+0x0); + + pr_info("CPU#%d finished\n", smp_processor_id()); +} + +static void __init loongson3_smp_setup(void) +{ + ipi_method_init(); + ipi_regaddrs_init(); + + if (cpu_has_csripi) + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); + else + xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(0)]); + + pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); + + cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; + cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; +} + +static void __init loongson3_prepare_cpus(unsigned int max_cpus) +{ + int i = 0; + + parse_acpi_topology(); + + for (i = 0; i < loongson_sysconf.nr_cpus; i++) { + set_cpu_present(i, true); + cpu_data[i].global_id = __cpu_logical_map[i]; + + if (cpu_has_csripi) + csr_mail_send(0, __cpu_logical_map[i], 0); + else + xconf_writeq(0, ipi_mailbox_buf[__cpu_logical_map[i]]+0x0); + } + + per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; +} + +/* + * Setup the PC, SP, and TP of a secondary processor and start it runing! + */ +static int loongson3_boot_secondary(int cpu, struct task_struct *idle) +{ + unsigned long entry; + + pr_info("Booting CPU#%d...\n", cpu); + + entry = __pa_symbol((unsigned long)&smpboot_entry); + cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); + cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); + + if (cpu_has_csripi) + csr_mail_send(entry, cpu_logical_map(cpu), 0); + else + xconf_writeq(entry, ipi_mailbox_buf[cpu_logical_map(cpu)]+0x0); + + loongson3_send_ipi_single(cpu, SMP_BOOT_CPU); + + return 0; +} + +#ifdef CONFIG_HOTPLUG_CPU + +static bool io_master(int cpu) +{ + int i, node, master; + + if (cpu == 0) + return true; + + for (i = 1; i < loongson_sysconf.nr_io_pics; i++) { + node = eiointc_get_node(i); + master = cpu_number_map(node * CORES_PER_EIO_NODE); + if (cpu == master) + return true; + } + + return false; +} + +static int loongson3_cpu_disable(void) +{ + unsigned long flags; + unsigned int cpu = smp_processor_id(); + + if (io_master(cpu)) + return -EBUSY; + +#ifdef CONFIG_NUMA + numa_remove_cpu(cpu); +#endif + set_cpu_online(cpu, false); + clear_cpu_sibling_map(cpu); + calculate_cpu_foreign_map(); + local_irq_save(flags); + irq_migrate_all_off_this_cpu(); + clear_csr_ecfg(ECFG0_IM); + local_irq_restore(flags); + local_flush_tlb_all(); + + return 0; +} + + +static void loongson3_cpu_die(unsigned int cpu) +{ + while (per_cpu(cpu_state, cpu) != CPU_DEAD) + cpu_relax(); + + mb(); +} + +void __noreturn arch_cpu_idle_dead(void) +{ + register long cpuid, core, node, count; + register void *addr, *base; + register void (*init_fn)(void); + + idle_task_exit(); + local_irq_enable(); + change_csr_ecfg(ECFG0_IM, ECFGF_IPI); + __this_cpu_write(cpu_state, CPU_DEAD); + + __smp_mb(); + __asm__ __volatile__( + " idle 0 \n" + " csrrd %[cpuid], 0x20 \n" + " andi %[cpuid], %[cpuid], 0x1ff \n" + " li.d %[base], 0x800000001fe01000 \n" + " andi %[core], %[cpuid], 0x3 \n" + " slli.w %[core], %[core], 8 \n" /* Get core id */ + " or %[base], %[base], %[core] \n" + " andi %[node], %[cpuid], 0x3c \n" + " slli.d %[node], %[node], 42 \n" /* Get node id */ + " or %[base], %[base], %[node] \n" + " ld.d %[init_fn], %[base], 0x20 \n" /* Get init PC */ + " nop \n" + : [core] "=&r" (core), [node] "=&r" (node), + [base] "=&r" (base), [cpuid] "=&r" (cpuid), + [count] "=&r" (count), [init_fn] "=&r" (addr) + : /* No Input */ + : "a1"); + + local_irq_disable(); + init_fn = __va(addr); + + init_fn(); + unreachable(); +} + +#endif + +const struct plat_smp_ops loongson3_smp_ops = { + .send_ipi_single = loongson3_send_ipi_single, + .send_ipi_mask = loongson3_send_ipi_mask, + .smp_setup = loongson3_smp_setup, + .prepare_cpus = loongson3_prepare_cpus, + .boot_secondary = loongson3_boot_secondary, + .init_secondary = loongson3_init_secondary, + .smp_finish = loongson3_smp_finish, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = loongson3_cpu_disable, + .cpu_die = loongson3_cpu_die, +#endif +}; + +/* + * Power management + */ +#ifdef CONFIG_PM + +static int loongson3_ipi_suspend(void) +{ + return 0; +} + +static void loongson3_ipi_resume(void) +{ + if (cpu_has_csripi) + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); + else + xconf_writel(0xffffffff, ipi_en_regs[cpu_logical_map(0)]); +} + +static struct syscore_ops loongson3_ipi_syscore_ops = { + .resume = loongson3_ipi_resume, + .suspend = loongson3_ipi_suspend, +}; + +/* + * Enable boot cpu ipi before enabling nonboot cpus + * during syscore_resume. + * */ +static int __init ipi_pm_init(void) +{ + register_syscore_ops(&loongson3_ipi_syscore_ops); + return 0; +} + +core_initcall(ipi_pm_init); +#endif diff --git a/arch/loongarch/mm/Makefile b/arch/loongarch/mm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fa6710785a5444ef2a2d2728130f07dd1b99ca98 --- /dev/null +++ b/arch/loongarch/mm/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux/LoongArch-specific parts of the memory manager. +# + +obj-y += init.o cache.o tlb.o tlbex.o extable.o \ + fault.o ioremap.o maccess.o mmap.o pgtable.o page.o + +obj-$(CONFIG_64BIT) += pgtable-64.o +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_KASAN) += kasan_init.o + +KASAN_SANITIZE_kasan_init.o := n diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c new file mode 100644 index 0000000000000000000000000000000000000000..51899ad6bf688e6b0a2c6f1bbefb8ec608c551c2 --- /dev/null +++ b/arch/loongarch/mm/cache.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +void cache_error_setup(void) +{ + extern char __weak except_vec_cex; + set_merr_handler(0x0, &except_vec_cex, 0x80); +} + +/* + * LoongArch maintains ICache/DCache coherency by hardware, + * we just need "ibar" to avoid instruction hazard here. + */ +void local_flush_icache_range(unsigned long start, unsigned long end) +{ + asm volatile ("\tibar 0\n"::); +} +EXPORT_SYMBOL(local_flush_icache_range); + +static void flush_cache_leaf(unsigned int leaf) +{ + int i, j, nr_nodes; + uint64_t addr = CSR_DMW0_BASE; + struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; + + nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; + + do { + for (i = 0; i < cdesc->sets; i++) { + for (j = 0; j < cdesc->ways; j++) { + flush_cache_line(leaf, addr); + addr++; + } + + addr -= cdesc->ways; + addr += cdesc->linesz; + } + addr += (1ULL << NODE_ADDRSPACE_SHIFT); + } while (--nr_nodes > 0); +} + +asmlinkage __visible void __flush_cache_all(void) +{ + int leaf; + struct cache_desc *cdesc = current_cpu_data.cache_leaves; + unsigned int cache_present = current_cpu_data.cache_leaves_present; + + leaf = cache_present - 1; + if (cache_inclusive(cdesc + leaf)) { + flush_cache_leaf(leaf); + return; + } + + for (leaf = 0; leaf < cache_present; leaf++) + flush_cache_leaf(leaf); +} + +#define L1IUPRE (1 << 0) +#define L1IUUNIFY (1 << 1) +#define L1DPRE (1 << 2) + +#define LXIUPRE (1 << 0) +#define LXIUUNIFY (1 << 1) +#define LXIUPRIV (1 << 2) +#define LXIUINCL (1 << 3) +#define LXDPRE (1 << 4) +#define LXDPRIV (1 << 5) +#define LXDINCL (1 << 6) + +#define populate_cache_properties(cfg0, cdesc, level, leaf) \ +do { \ + unsigned int cfg1; \ + \ + cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ + if (level == 1) { \ + cdesc->flags |= CACHE_PRIVATE; \ + } else { \ + if (cfg0 & LXIUPRIV) \ + cdesc->flags |= CACHE_PRIVATE; \ + if (cfg0 & LXIUINCL) \ + cdesc->flags |= CACHE_INCLUSIVE; \ + } \ + cdesc->level = level; \ + cdesc->flags |= CACHE_PRESENT; \ + cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ + cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ + cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ + cdesc++; leaf++; \ +} while (0) + +void cpu_cache_init(void) +{ + unsigned int leaf = 0, level = 1; + unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); + struct cache_desc *cdesc = current_cpu_data.cache_leaves; + + if (config & L1IUPRE) { + if (config & L1IUUNIFY) + cdesc->type = CACHE_TYPE_UNIFIED; + else + cdesc->type = CACHE_TYPE_INST; + populate_cache_properties(config, cdesc, level, leaf); + } + + if (config & L1DPRE) { + cdesc->type = CACHE_TYPE_DATA; + populate_cache_properties(config, cdesc, level, leaf); + } + + config = config >> 3; + for (level = 2; level <= CACHE_LEVEL_MAX; level++) { + if (!config) + break; + + if (config & LXIUPRE) { + if (config & LXIUUNIFY) + cdesc->type = CACHE_TYPE_UNIFIED; + else + cdesc->type = CACHE_TYPE_INST; + populate_cache_properties(config, cdesc, level, leaf); + } + + if (config & LXDPRE) { + cdesc->type = CACHE_TYPE_DATA; + populate_cache_properties(config, cdesc, level, leaf); + } + + config = config >> 7; + } + + BUG_ON(leaf > CACHE_LEAVES_MAX); + + current_cpu_data.cache_leaves_present = leaf; + current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; +} diff --git a/arch/loongarch/mm/extable.c b/arch/loongarch/mm/extable.c new file mode 100644 index 0000000000000000000000000000000000000000..eead3f7bf3f4c6c0edd4762c15d874d507230974 --- /dev/null +++ b/arch/loongarch/mm/extable.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + +static inline unsigned long +get_ex_fixup(const struct exception_table_entry *ex) +{ + return ((unsigned long)&ex->fixup + ex->fixup); +} + +static inline void regs_set_gpr(struct pt_regs *regs, + unsigned int offset, unsigned long val) +{ + if (offset && offset <= MAX_REG_OFFSET) + *(unsigned long *)((unsigned long)regs + offset) = val; +} + +static bool ex_handler_fixup(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + regs->csr_era = get_ex_fixup(ex); + + return true; +} + +static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data); + + regs_set_gpr(regs, reg_err * sizeof(unsigned long), -EFAULT); + regs_set_gpr(regs, reg_zero * sizeof(unsigned long), 0); + regs->csr_era = get_ex_fixup(ex); + + return true; +} + +bool fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_exception_tables(exception_era(regs)); + if (!ex) + return false; + + switch (ex->type) { + case EX_TYPE_FIXUP: + return ex_handler_fixup(ex, regs); + case EX_TYPE_UACCESS_ERR_ZERO: + return ex_handler_uaccess_err_zero(ex, regs); + } + + BUG(); +} diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c new file mode 100644 index 0000000000000000000000000000000000000000..e53d7f24801e2e51cbe4cfa1569d97af35d3b3e3 --- /dev/null +++ b/arch/loongarch/mm/fault.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 1995 - 2000 by Ralf Baechle + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +int show_unhandled_signals = 1; + +static void __kprobes no_context(struct pt_regs *regs, unsigned long address) +{ + const int field = sizeof(unsigned long) * 2; + + /* Are we prepared to handle this kernel fault? */ + if (fixup_exception(regs)) + return; + + /* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + bust_spinlocks(1); + + pr_alert("CPU %d Unable to handle kernel paging request at " + "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", + raw_smp_processor_id(), field, address, field, regs->csr_era, + field, regs->regs[1]); + die("Oops", regs); +} + +static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address) +{ + /* + * We ran out of memory, call the OOM killer, and return the userspace + * (which will retry the fault, or kill us if we got oom-killed). + */ + if (!user_mode(regs)) { + no_context(regs, address); + return; + } + pagefault_out_of_memory(); +} + +static void __kprobes do_sigbus(struct pt_regs *regs, + unsigned long write, unsigned long address, int si_code) +{ + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) { + no_context(regs, address); + return; + } + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + current->thread.csr_badvaddr = address; + current->thread.trap_nr = read_csr_excode(); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); +} + +static void __kprobes do_sigsegv(struct pt_regs *regs, + unsigned long write, unsigned long address, int si_code) +{ + const int field = sizeof(unsigned long) * 2; + static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) { + no_context(regs, address); + return; + } + + /* User mode accesses just cause a SIGSEGV */ + current->thread.csr_badvaddr = address; + if (!write) + current->thread.error_code = 1; + else + current->thread.error_code = 2; + current->thread.trap_nr = read_csr_excode(); + + if (show_unhandled_signals && + unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { + pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", + current->comm, + write ? "write access to" : "read access from", + field, address); + pr_info("era = %0*lx in", field, + (unsigned long) regs->csr_era); + print_vma_addr(KERN_CONT " ", regs->csr_era); + pr_cont("\n"); + pr_info("ra = %0*lx in", field, + (unsigned long) regs->regs[1]); + print_vma_addr(KERN_CONT " ", regs->regs[1]); + pr_cont("\n"); + } + force_sig_fault(SIGSEGV, si_code, (void __user *)address); +} + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +static void __kprobes __do_page_fault(struct pt_regs *regs, + unsigned long write, unsigned long address) +{ + int si_code = SEGV_MAPERR; + unsigned int flags = FAULT_FLAG_DEFAULT; + struct task_struct *tsk = current; + struct mm_struct *mm = tsk->mm; + struct vm_area_struct *vma = NULL; + vm_fault_t fault; + +#ifdef CONFIG_KPROBES + /* + * This is to notify the fault handler of the kprobes. + */ + if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1, + current->thread.trap_nr, SIGSEGV) == NOTIFY_STOP) + return; +#endif + + /* + * We fault-in kernel-space virtual memory on-demand. The + * 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. + */ + if (address & __UA_LIMIT) { + if (!user_mode(regs)) + no_context(regs, address); + else + do_sigsegv(regs, write, address, si_code); + return; + } + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (faulthandler_disabled() || !mm) { + do_sigsegv(regs, write, address, si_code); + return; + } + + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); +retry: + mmap_read_lock(mm); + vma = find_vma(mm, address); + if (!vma) + goto bad_area; + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (!expand_stack(vma, address)) + goto good_area; +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + mmap_read_unlock(mm); + do_sigsegv(regs, write, address, si_code); + return; + +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + si_code = SEGV_ACCERR; + + if (write) { + flags |= FAULT_FLAG_WRITE; + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) + goto bad_area; + if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) + goto bad_area; + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, address, flags, regs); + + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, address); + return; + } + + if (unlikely(fault & VM_FAULT_RETRY)) { + flags |= FAULT_FLAG_TRIED; + + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + goto retry; + } + if (unlikely(fault & VM_FAULT_ERROR)) { + mmap_read_unlock(mm); + if (fault & VM_FAULT_OOM) { + do_out_of_memory(regs, address); + return; + } else if (fault & VM_FAULT_SIGSEGV) { + do_sigsegv(regs, write, address, si_code); + return; + } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { + do_sigbus(regs, write, address, si_code); + return; + } + BUG(); + } + + mmap_read_unlock(mm); +} + +asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, + unsigned long write, unsigned long address) +{ + irqentry_state_t state = irqentry_enter(regs); + + /* Enable interrupt if enabled in parent context */ + if (likely(regs->csr_prmd & CSR_PRMD_PIE)) + local_irq_enable(); + + __do_page_fault(regs, write, address); + + local_irq_disable(); + + irqentry_exit(regs, state); +} diff --git a/arch/loongarch/mm/hugetlbpage.c b/arch/loongarch/mm/hugetlbpage.c new file mode 100644 index 0000000000000000000000000000000000000000..cf4f81abc5a7c2c04ecdd54c90bca9b5b5e1321b --- /dev/null +++ b/arch/loongarch/mm/hugetlbpage.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (pud) + pte = (pte_t *)pmd_alloc(mm, pud, addr); + + return pte; +} + +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + + pgd = pgd_offset(mm, addr); + if (pgd_present(*pgd)) { + p4d = p4d_offset(pgd, addr); + if (p4d_present(*p4d)) { + pud = pud_offset(p4d, addr); + if (pud_present(*pud)) + pmd = pmd_offset(pud, addr); + } + } + return (pte_t *) pmd; +} + +int pmd_huge(pmd_t pmd) +{ + return (pmd_val(pmd) & _PAGE_HUGE) != 0; +} +EXPORT_SYMBOL(pmd_huge); + +int pud_huge(pud_t pud) +{ + return (pud_val(pud) & _PAGE_HUGE) != 0; +} +EXPORT_SYMBOL(pud_huge); + +uint64_t pmd_to_entrylo(unsigned long pmd_val) +{ + uint64_t val; + /* PMD as PTE. Must be huge page */ + if (!pmd_huge(__pmd(pmd_val))) + panic("%s", __func__); + + val = pmd_val ^ _PAGE_HUGE; + val |= ((val & _PAGE_HGLOBAL) >> + (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); + + return val; +} diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c new file mode 100644 index 0000000000000000000000000000000000000000..48b0cac9b3cb0f823a8e85f5494fb0636ebe7f17 --- /dev/null +++ b/arch/loongarch/mm/init.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; +EXPORT_SYMBOL(empty_zero_page); + +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma) +{ + void *vfrom, *vto; + + vto = kmap_atomic(to); + vfrom = kmap_atomic(from); + copy_page(vto, vfrom); + kunmap_atomic(vfrom); + kunmap_atomic(vto); + /* Make sure this page is cleared on other CPU's too before using it */ + smp_wmb(); +} + +int __ref page_is_ram(unsigned long pfn) +{ + unsigned long addr = PFN_PHYS(pfn); + + return memblock_is_memory(addr) && !memblock_is_reserved(addr); +} + +#ifndef CONFIG_NEED_MULTIPLE_NODES +void __init paging_init(void) +{ + unsigned long max_zone_pfns[MAX_NR_ZONES]; + +#ifdef CONFIG_ZONE_DMA + max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; +#endif +#ifdef CONFIG_ZONE_DMA32 + max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; +#endif + max_zone_pfns[ZONE_NORMAL] = max_low_pfn; + + free_area_init(max_zone_pfns); +} + +void __init mem_init(void) +{ + max_mapnr = max_low_pfn; + high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); + + memblock_free_all(); + mem_init_print_info(NULL); +} +#endif /* !CONFIG_NEED_MULTIPLE_NODES */ + +void __ref free_initmem(void) +{ + free_initmem_default(POISON_FREE_INITMEM); +} + +#ifdef CONFIG_MEMORY_HOTPLUG +int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) +{ + unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + int ret; + + ret = __add_pages(nid, start_pfn, nr_pages, params); + + if (ret) + printk("%s: Problem encountered in __add_pages() as ret=%d\n", + __func__, ret); + + return ret; +} + +void arch_remove_memory(int nid, u64 start, + u64 size, struct vmem_altmap *altmap) +{ + unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + struct page *page = pfn_to_page(start_pfn); + + /* With altmap the first mapped page is offset from @start */ + if (altmap) + page += vmem_altmap_offset(altmap); + __remove_pages(start_pfn, nr_pages, altmap); +} + +#ifdef CONFIG_NUMA +int memory_add_physaddr_to_nid(u64 start) +{ + return pa_to_nid(start); +} +EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); +#endif +#endif + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +void __meminit arch_vmemmap_verify(pte_t *pte, int node, + unsigned long start, unsigned long end) +{ + unsigned long pfn = pte_pfn(*pte); + int actual_node = early_pfn_to_nid(pfn); + + if (node_distance(actual_node, node) > LOCAL_DISTANCE) + pr_warn("[%lx-%lx] potential offnode page_structs\n", + start, end - 1); +} + +void * __meminit arch_vmemmap_alloc_block_zero(unsigned long size, int node) +{ + void *p = vmemmap_alloc_block(size, node); + + if (!p) + return NULL; + memset(p, 0, size); + + return p; +} + +pte_t * __meminit arch_vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + if (pte_none(*pte)) { + pte_t entry; + void *p = arch_vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); + } + return pte; +} + +pmd_t * __meminit arch_vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) +{ + pmd_t *pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + void *p = arch_vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; + pmd_populate_kernel(&init_mm, pmd, p); + } + return pmd; +} + +pud_t * __meminit arch_vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) +{ + pud_t *pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + void *p = arch_vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; +#ifndef __PAGETABLE_PMD_FOLDED + pmd_init((unsigned long)p, (unsigned long)invalid_pte_table); +#endif + pud_populate(&init_mm, pud, p); + } + return pud; +} + +p4d_t * __meminit arch_vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + void *p = arch_vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; +#ifndef __PAGETABLE_PUD_FOLDED + pud_init((unsigned long)p, (unsigned long)invalid_pmd_table); +#endif + p4d_populate(&init_mm, p4d, p); + } + return p4d; +} + +pgd_t * __meminit arch_vmemmap_pgd_populate(unsigned long addr, int node) +{ + pgd_t *pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) { + void *p = arch_vmemmap_alloc_block_zero(PAGE_SIZE, node); + if (!p) + return NULL; + pgd_populate(&init_mm, pgd, p); + } + return pgd; +} + +int __meminit arch_vmemmap_populate_basepages(unsigned long start, + unsigned long end, int node) +{ + unsigned long addr = start; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + for (; addr < end; addr += PAGE_SIZE) { + pgd = arch_vmemmap_pgd_populate(addr, node); + if (!pgd) + return -ENOMEM; + p4d = arch_vmemmap_p4d_populate(pgd, addr, node); + if (!p4d) + return -ENOMEM; + pud = arch_vmemmap_pud_populate(p4d, addr, node); + if (!pud) + return -ENOMEM; + pmd = arch_vmemmap_pmd_populate(pud, addr, node); + if (!pmd) + return -ENOMEM; + pte = arch_vmemmap_pte_populate(pmd, addr, node); + if (!pte) + return -ENOMEM; + arch_vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); + } + + return 0; +} + +int __meminit arch_vmemmap_populate_hugepages(unsigned long start, + unsigned long end, int node) +{ + unsigned long addr = start; + unsigned long next; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + for (addr = start; addr < end; addr = next) { + next = pmd_addr_end(addr, end); + + pgd = arch_vmemmap_pgd_populate(addr, node); + if (!pgd) + return -ENOMEM; + p4d = arch_vmemmap_p4d_populate(pgd, addr, node); + if (!p4d) + return -ENOMEM; + pud = arch_vmemmap_pud_populate(p4d, addr, node); + if (!pud) + return -ENOMEM; + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + void *p = NULL; + + p = arch_vmemmap_alloc_block_zero(PMD_SIZE, node); + if (p) { + pmd_t entry; + + entry = pfn_pmd(virt_to_pfn(p), PAGE_KERNEL); + pmd_val(entry) |= _PAGE_HUGE | _PAGE_HGLOBAL; + set_pmd_at(&init_mm, addr, pmd, entry); + + continue; + } + } else if (pmd_val(*pmd) & _PAGE_HUGE) { + arch_vmemmap_verify((pte_t *)pmd, node, addr, next); + continue; + } + if (arch_vmemmap_populate_basepages(addr, next, node)) + return -ENOMEM; + } + + return 0; +} + +int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, + struct vmem_altmap *altmap) +{ + return arch_vmemmap_populate_hugepages(start, end, node); +} +void vmemmap_free(unsigned long start, unsigned long end, + struct vmem_altmap *altmap) +{ +} +#endif + +pte_t * __init populate_kernel_pte(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud; + pmd_t *pmd; + + if (p4d_none(*p4d)) { + pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pud) + panic("%s: Failed to allocate memory\n", __func__); + p4d_populate(&init_mm, p4d, pud); +#ifndef __PAGETABLE_PUD_FOLDED + pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table); +#endif + } + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pmd) + panic("%s: Failed to allocate memory\n", __func__); + pud_populate(&init_mm, pud, pmd); +#ifndef __PAGETABLE_PMD_FOLDED + pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table); +#endif + } + + pmd = pmd_offset(pud, addr); + if (!pmd_present(*pmd)) { + pte_t *pte; + + pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate memory\n", __func__); + pmd_populate_kernel(&init_mm, pmd, pte); + } + + return pte_offset_kernel(pmd, addr); +} + +void __init __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *ptep; + + BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + ptep = populate_kernel_pte(addr); + if (!pte_none(*ptep)) { + pte_ERROR(*ptep); + return; + } + + if (pgprot_val(flags)) + set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); + else { + pte_clear(&init_mm, addr, ptep); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + } +} + +/* + * Align swapper_pg_dir in to 64K, allows its address to be loaded + * with a single LUI instruction in the TLB handlers. If we used + * __aligned(64K), its size would get rounded up to the alignment + * size, and waste space. So we place it in its own section and align + * it in the linker script. + */ +pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); + +pgd_t invalid_pg_dir[_PTRS_PER_PGD] __page_aligned_bss; +#ifndef __PAGETABLE_PUD_FOLDED +pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss; +EXPORT_SYMBOL(invalid_pud_table); +#endif +#ifndef __PAGETABLE_PMD_FOLDED +pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss; +EXPORT_SYMBOL(invalid_pmd_table); +#endif +pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; +EXPORT_SYMBOL(invalid_pte_table); diff --git a/arch/loongarch/mm/ioremap.c b/arch/loongarch/mm/ioremap.c new file mode 100644 index 0000000000000000000000000000000000000000..c635a6afc6f7f7a71ff5787b7bede16f4583f361 --- /dev/null +++ b/arch/loongarch/mm/ioremap.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size) +{ + return ((void __iomem *)TO_CACHE(phys_addr)); +} + +void __init early_iounmap(void __iomem *addr, unsigned long size) +{ + +} + +void *early_memremap_ro(resource_size_t phys_addr, unsigned long size) +{ + return early_memremap(phys_addr, size); +} + +void *early_memremap_prot(resource_size_t phys_addr, unsigned long size, + unsigned long prot_val) +{ + return early_memremap(phys_addr, size); +} diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c new file mode 100644 index 0000000000000000000000000000000000000000..b3eaceac14895757c14edd258ba5d31b93cb013f --- /dev/null +++ b/arch/loongarch/mm/kasan_init.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#define pr_fmt(fmt) "kasan: " fmt +#include +#include +#include + +#include +#include + +static pgd_t kasan_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); + +#define __pgd_none(early, pgd) (early ? (pgd_val(pgd) == 0) : \ +(__pa(pgd_val(pgd)) == (unsigned long)__pa(kasan_early_shadow_pmd))) + +#define __pmd_none(early, pmd) (early ? (pmd_val(pmd) == 0) : \ +(__pa(pmd_val(pmd)) == (unsigned long)__pa(kasan_early_shadow_pte))) + +#define __pte_none(early, pte) (early ? pte_none(pte) : \ +((pte_val(pte) & _PFN_MASK) == (unsigned long)__pa(kasan_early_shadow_page))) + +bool kasan_early_stage = true; + +void *kasan_mem_to_shadow(const void *addr) +{ + if (kasan_early_stage) { + return (void *)(kasan_early_shadow_page); + } else { + unsigned long maddr = (unsigned long)addr; + unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; + unsigned long offset = 0; + + if (maddr >= FIXADDR_START) + return (void *)(kasan_early_shadow_page); + + maddr &= XRANGE_SHADOW_MASK; + switch (xrange) { + case XKPRANGE_CC_SEG: + offset = XKPRANGE_CC_SHADOW_OFFSET; + break; + case XKPRANGE_UC_SEG: + offset = XKPRANGE_UC_SHADOW_OFFSET; + break; + case XKVRANGE_VC_SEG: + offset = XKVRANGE_VC_SHADOW_OFFSET; + break; + default: + WARN_ON(1); + return NULL; + } + + return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); + } +} + +const void *kasan_shadow_to_mem(const void *shadow_addr) +{ + unsigned long addr = (unsigned long)shadow_addr; + + if (unlikely(addr > KASAN_SHADOW_END) || + unlikely(addr < KASAN_SHADOW_START)) { + WARN_ON(1); + return NULL; + } + + if (addr >= XKVRANGE_VC_SHADOW_OFFSET) + return (void *)(((addr - XKVRANGE_VC_SHADOW_OFFSET) << KASAN_SHADOW_SCALE_SHIFT) + XKVRANGE_VC_START); + else if (addr >= XKPRANGE_UC_SHADOW_OFFSET) + return (void *)(((addr - XKPRANGE_UC_SHADOW_OFFSET) << KASAN_SHADOW_SCALE_SHIFT) + XKPRANGE_UC_START); + else if (addr >= XKPRANGE_CC_SHADOW_OFFSET) + return (void *)(((addr - XKPRANGE_CC_SHADOW_OFFSET) << KASAN_SHADOW_SCALE_SHIFT) + XKPRANGE_CC_START); + else { + WARN_ON(1); + return NULL; + } +} + +/* + * Alloc memory for shadow memory page table. + */ +static phys_addr_t __init kasan_alloc_zeroed_page(int node) +{ + void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE, + __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_ACCESSIBLE, node); + return __pa(p); +} + +static pte_t *kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, + bool early) +{ + if (__pmd_none(early, READ_ONCE(*pmdp))) { + phys_addr_t pte_phys = early ? + __pa_symbol(kasan_early_shadow_pte) + : kasan_alloc_zeroed_page(node); + if (!early) + memcpy(__va(pte_phys), kasan_early_shadow_pte, + sizeof(kasan_early_shadow_pte)); + + pmd_populate_kernel(NULL, pmdp, (pte_t *)__va(pte_phys)); + } + + return pte_offset_kernel(pmdp, addr); +} + +static inline void kasan_set_pgd(pgd_t *pgdp, pgd_t pgdval) +{ + WRITE_ONCE(*pgdp, pgdval); +} + +static pmd_t *kasan_pmd_offset(pgd_t *pgdp, unsigned long addr, int node, + bool early) +{ + if (__pgd_none(early, READ_ONCE(*pgdp))) { + phys_addr_t pmd_phys = early ? + __pa_symbol(kasan_early_shadow_pmd) + : kasan_alloc_zeroed_page(node); + if (!early) + memcpy(__va(pmd_phys), kasan_early_shadow_pmd, + sizeof(kasan_early_shadow_pmd)); + kasan_set_pgd(pgdp, __pgd((unsigned long)__va(pmd_phys))); + } + + return (pmd_t *)((pmd_t *)pgd_val(*pgdp) + pmd_index(addr)); +} + +static void kasan_pte_populate(pmd_t *pmdp, unsigned long addr, + unsigned long end, int node, bool early) +{ + unsigned long next; + pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); + + do { + phys_addr_t page_phys = early ? + __pa_symbol(kasan_early_shadow_page) + : kasan_alloc_zeroed_page(node); + next = addr + PAGE_SIZE; + set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL)); + } while (ptep++, addr = next, addr != end && __pte_none(early, READ_ONCE(*ptep))); +} + +static void kasan_pmd_populate(pgd_t *pgdp, unsigned long addr, + unsigned long end, int node, bool early) +{ + unsigned long next; + pmd_t *pmdp = kasan_pmd_offset(pgdp, addr, node, early); + + do { + next = pmd_addr_end(addr, end); + kasan_pte_populate(pmdp, addr, next, node, early); + } while (pmdp++, addr = next, addr != end && __pmd_none(early, READ_ONCE(*pmdp))); +} + +static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, + int node, bool early) +{ + unsigned long next; + pgd_t *pgdp; + + pgdp = pgd_offset_k(addr); + + do { + next = pgd_addr_end(addr, end); + kasan_pmd_populate(pgdp, addr, next, node, early); + } while (pgdp++, addr = next, addr != end); + +} + +asmlinkage void __init kasan_early_init(void) +{ + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE)); + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE)); +} + +/* Set up full kasan mappings, ensuring that the mapped pages are zeroed */ +static void __init kasan_map_populate(unsigned long start, unsigned long end, + int node) +{ + kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false); +} + +static void __init clear_pgds(unsigned long start, + unsigned long end) +{ + for (; start < end; start += PGDIR_SIZE) + kasan_set_pgd((pgd_t *)pgd_offset_k(start), __pgd(0)); +} + +void __init kasan_init(void) +{ + u64 i; + phys_addr_t pa_start, pa_end; + + /* + * PGD was populated as invalid_pmd_table or invalid_pud_table + * in pagetable_init() which depends on how many levels of page + * table you are using, but we had to clean the gpd of kasan + * shadow memory, as the pgd value is none-zero. + * The assertion pgd_none is going to be false and the formal populate + * afterwards is not going to create any new pgd at all. + */ + memcpy(kasan_pg_dir, swapper_pg_dir, sizeof(kasan_pg_dir)); + csr_write64(__pa_symbol(kasan_pg_dir), LOONGARCH_CSR_PGDH); + local_flush_tlb_all(); + + clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); + + /* Maps everything to a single page of zeroes */ + kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true); + + kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), + kasan_mem_to_shadow((void *)VMEMMAP_END)); + + kasan_early_stage = false; + + /* Populate the linear mapping */ + for_each_mem_range(i, &pa_start, &pa_end) { + void *start = (void *)phys_to_virt(pa_start); + void *end = (void *)phys_to_virt(pa_end); + + if (start >= end) + break; + + kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), + (unsigned long)kasan_mem_to_shadow(end), NUMA_NO_NODE); + } + + /* Populate modules mapping */ + kasan_map_populate((unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR), + (unsigned long)kasan_mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); + /* + * KAsan may reuse the contents of kasan_zero_pte directly, so we + * should make sure that it maps the zero page read-only. + */ + for (i = 0; i < PTRS_PER_PTE; i++) + set_pte(&kasan_early_shadow_pte[i], + pfn_pte(__phys_to_pfn(__pa_symbol(kasan_early_shadow_page)), + PAGE_KERNEL_RO)); + + memset(kasan_early_shadow_page, 0, PAGE_SIZE); + csr_write64(__pa_symbol(swapper_pg_dir), LOONGARCH_CSR_PGDH); + local_flush_tlb_all(); + + /* At this point kasan is fully initialized. Enable error messages */ + init_task.kasan_depth = 0; + pr_info("KernelAddressSanitizer initialized.\n"); +} diff --git a/arch/loongarch/mm/maccess.c b/arch/loongarch/mm/maccess.c new file mode 100644 index 0000000000000000000000000000000000000000..58173842c6bebd5048cda593707cb326009f8305 --- /dev/null +++ b/arch/loongarch/mm/maccess.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) +{ + /* highest bit set means kernel space */ + return (unsigned long)unsafe_src >> (BITS_PER_LONG - 1); +} diff --git a/arch/loongarch/mm/mmap.c b/arch/loongarch/mm/mmap.c new file mode 100644 index 0000000000000000000000000000000000000000..6e6ac5c214142ad750f926406cb09251e62707fd --- /dev/null +++ b/arch/loongarch/mm/mmap.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + +#define SHM_ALIGN_MASK (SHMLBA - 1) + +#define COLOUR_ALIGN(addr, pgoff) \ + ((((addr) + SHM_ALIGN_MASK) & ~SHM_ALIGN_MASK) \ + + (((pgoff) << PAGE_SHIFT) & SHM_ALIGN_MASK)) + +enum mmap_allocation_direction {UP, DOWN}; + +static unsigned long arch_get_unmapped_area_common(struct file *filp, + unsigned long addr0, unsigned long len, unsigned long pgoff, + unsigned long flags, enum mmap_allocation_direction dir) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long addr = addr0; + int do_color_align; + struct vm_unmapped_area_info info; + + if (unlikely(len > TASK_SIZE)) + return -ENOMEM; + + if (flags & MAP_FIXED) { + /* Even MAP_FIXED mappings must reside within TASK_SIZE */ + if (TASK_SIZE - len < addr) + return -EINVAL; + + /* + * We do not accept a shared mapping if it would violate + * cache aliasing constraints. + */ + if ((flags & MAP_SHARED) && + ((addr - (pgoff << PAGE_SHIFT)) & SHM_ALIGN_MASK)) + return -EINVAL; + return addr; + } + + do_color_align = 0; + if (filp || (flags & MAP_SHARED)) + do_color_align = 1; + + /* requesting a specific address */ + if (addr) { + if (do_color_align) + addr = COLOUR_ALIGN(addr, pgoff); + else + addr = PAGE_ALIGN(addr); + + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vm_start_gap(vma))) + return addr; + } + + info.length = len; + info.align_mask = do_color_align ? (PAGE_MASK & SHM_ALIGN_MASK) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + + if (dir == DOWN) { + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.low_limit = PAGE_SIZE; + info.high_limit = mm->mmap_base; + addr = vm_unmapped_area(&info); + + if (!(addr & ~PAGE_MASK)) + return addr; + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + } + + info.flags = 0; + info.low_limit = mm->mmap_base; + info.high_limit = TASK_SIZE; + return vm_unmapped_area(&info); +} + +unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + return arch_get_unmapped_area_common(filp, + addr0, len, pgoff, flags, UP); +} + +/* + * There is no need to export this but sched.h declares the function as + * extern so making it static here results in an error. + */ +unsigned long arch_get_unmapped_area_topdown(struct file *filp, + unsigned long addr0, unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + return arch_get_unmapped_area_common(filp, + addr0, len, pgoff, flags, DOWN); +} + +int __virt_addr_valid(volatile void *kaddr) +{ + unsigned long vaddr = (unsigned long)kaddr; + + if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base)) + return 0; + + return pfn_valid(PFN_DOWN(PHYSADDR(kaddr))); +} +EXPORT_SYMBOL_GPL(__virt_addr_valid); + +/* + * You really shouldn't be using read() or write() on /dev/mem. This might go + * away in the future. + */ +int valid_phys_addr_range(phys_addr_t addr, size_t size) +{ + /* + * Check whether addr is covered by a memory region without the + * MEMBLOCK_NOMAP attribute, and whether that region covers the + * entire range. In theory, this could lead to false negatives + * if the range is covered by distinct but adjacent memory regions + * that only differ in other attributes. However, few of such + * attributes have been defined, and it is debatable whether it + * follows that /dev/mem read() calls should be able traverse + * such boundaries. + */ + return memblock_is_region_memory(addr, size) && memblock_is_map_memory(addr); +} + +/* + * Do not allow /dev/mem mappings beyond the supported physical range. + */ +int valid_mmap_phys_addr_range(unsigned long pfn, size_t size) +{ + return !(((pfn << PAGE_SHIFT) + size) & ~(GENMASK_ULL(cpu_pabits, 0))); +} diff --git a/arch/loongarch/mm/page.S b/arch/loongarch/mm/page.S new file mode 100644 index 0000000000000000000000000000000000000000..bf77d2815449c11c4692bc3e910325e2f6898c13 --- /dev/null +++ b/arch/loongarch/mm/page.S @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + + .align 5 +SYM_FUNC_START(clear_page) + lu12i.w t0, 1 << (PAGE_SHIFT - 12) + add.d t0, t0, a0 +1: + st.d zero, a0, 0 + st.d zero, a0, 8 + st.d zero, a0, 16 + st.d zero, a0, 24 + st.d zero, a0, 32 + st.d zero, a0, 40 + st.d zero, a0, 48 + st.d zero, a0, 56 + addi.d a0, a0, 128 + st.d zero, a0, -64 + st.d zero, a0, -56 + st.d zero, a0, -48 + st.d zero, a0, -40 + st.d zero, a0, -32 + st.d zero, a0, -24 + st.d zero, a0, -16 + st.d zero, a0, -8 + bne t0, a0, 1b + + jirl $r0, ra, 0 +SYM_FUNC_END(clear_page) +EXPORT_SYMBOL(clear_page) + +.align 5 +SYM_FUNC_START(copy_page) + lu12i.w t8, 1 << (PAGE_SHIFT - 12) + add.d t8, t8, a0 +1: + ld.d t0, a1, 0 + ld.d t1, a1, 8 + ld.d t2, a1, 16 + ld.d t3, a1, 24 + ld.d t4, a1, 32 + ld.d t5, a1, 40 + ld.d t6, a1, 48 + ld.d t7, a1, 56 + + st.d t0, a0, 0 + st.d t1, a0, 8 + ld.d t0, a1, 64 + ld.d t1, a1, 72 + st.d t2, a0, 16 + st.d t3, a0, 24 + ld.d t2, a1, 80 + ld.d t3, a1, 88 + st.d t4, a0, 32 + st.d t5, a0, 40 + ld.d t4, a1, 96 + ld.d t5, a1, 104 + st.d t6, a0, 48 + st.d t7, a0, 56 + ld.d t6, a1, 112 + ld.d t7, a1, 120 + addi.d a0, a0, 128 + addi.d a1, a1, 128 + + st.d t0, a0, -64 + st.d t1, a0, -56 + st.d t2, a0, -48 + st.d t3, a0, -40 + st.d t4, a0, -32 + st.d t5, a0, -24 + st.d t6, a0, -16 + st.d t7, a0, -8 + + bne t8, a0, 1b + jirl $r0, ra, 0 +SYM_FUNC_END(copy_page) +EXPORT_SYMBOL(copy_page) diff --git a/arch/loongarch/mm/pgtable-64.c b/arch/loongarch/mm/pgtable-64.c new file mode 100644 index 0000000000000000000000000000000000000000..dcf7e79cd5dc176ea7e7078e7154e0f7b5e7a702 --- /dev/null +++ b/arch/loongarch/mm/pgtable-64.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include + +void pgd_init(unsigned long page) +{ + unsigned long *p, *end; + unsigned long entry; + +#if !defined(__PAGETABLE_PUD_FOLDED) + entry = (unsigned long)invalid_pud_table; +#elif !defined(__PAGETABLE_PMD_FOLDED) + entry = (unsigned long)invalid_pmd_table; +#else + entry = (unsigned long)invalid_pte_table; +#endif + + p = (unsigned long *) page; + end = p + PTRS_PER_PGD; + + do { + p[0] = entry; + p[1] = entry; + p[2] = entry; + p[3] = entry; + p[4] = entry; + p += 8; + p[-3] = entry; + p[-2] = entry; + p[-1] = entry; + } while (p != end); +} +EXPORT_SYMBOL_GPL(pgd_init); + +#ifndef __PAGETABLE_PMD_FOLDED +void pmd_init(unsigned long addr, unsigned long pagetable) +{ + unsigned long *p, *end; + + p = (unsigned long *) addr; + end = p + PTRS_PER_PMD; + + do { + p[0] = pagetable; + p[1] = pagetable; + p[2] = pagetable; + p[3] = pagetable; + p[4] = pagetable; + p += 8; + p[-3] = pagetable; + p[-2] = pagetable; + p[-1] = pagetable; + } while (p != end); +} +EXPORT_SYMBOL_GPL(pmd_init); +#endif + +#ifndef __PAGETABLE_PUD_FOLDED +void pud_init(unsigned long addr, unsigned long pagetable) +{ + unsigned long *p, *end; + + p = (unsigned long *)addr; + end = p + PTRS_PER_PUD; + + do { + p[0] = pagetable; + p[1] = pagetable; + p[2] = pagetable; + p[3] = pagetable; + p[4] = pagetable; + p += 8; + p[-3] = pagetable; + p[-2] = pagetable; + p[-1] = pagetable; + } while (p != end); +} +#endif + +pmd_t mk_pmd(struct page *page, pgprot_t prot) +{ + pmd_t pmd; + + pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); + + return pmd; +} + +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; + flush_tlb_all(); +} + +void __init pagetable_init(void) +{ + unsigned long vaddr; + + /* Initialize the entire pgd. */ + pgd_init((unsigned long)swapper_pg_dir); + pgd_init((unsigned long)invalid_pg_dir); +#ifndef __PAGETABLE_PUD_FOLDED + pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table); +#endif +#ifndef __PAGETABLE_PMD_FOLDED + pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table); +#endif + /* + * Fixed mappings: + */ + vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; +} diff --git a/arch/loongarch/mm/pgtable.c b/arch/loongarch/mm/pgtable.c new file mode 100644 index 0000000000000000000000000000000000000000..648d8d59cea6b52c50c6e714a79f1b6ca6eeaa71 --- /dev/null +++ b/arch/loongarch/mm/pgtable.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *ret, *init; + + ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER); + if (ret) { + init = pgd_offset(&init_mm, 0UL); + pgd_init((unsigned long)ret); + memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + } + + return ret; +} +EXPORT_SYMBOL_GPL(pgd_alloc); diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c new file mode 100644 index 0000000000000000000000000000000000000000..cbecddd88bd0aee992790150a33bbf4946bd3e30 --- /dev/null +++ b/arch/loongarch/mm/tlb.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +extern void *exception_table[]; + +void local_flush_tlb_all(void) +{ + invtlb_all(INVTLB_CURRENT_ALL, 0, 0); +} +EXPORT_SYMBOL(local_flush_tlb_all); + +void local_flush_tlb_user(void) +{ + invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0); +} +EXPORT_SYMBOL(local_flush_tlb_user); + +void local_flush_tlb_kernel(void) +{ + invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0); +} +EXPORT_SYMBOL(local_flush_tlb_kernel); + +/* All entries common to a mm share an asid. To effectively flush + these entries, we just bump the asid. */ +void local_flush_tlb_mm(struct mm_struct *mm) +{ + int cpu; + + preempt_disable(); + + cpu = smp_processor_id(); + + if (asid_valid(mm, cpu)) { + drop_mmu_context(mm, cpu); + } else { + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + } + + preempt_enable(); +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + int cpu = smp_processor_id(); + + if (asid_valid(mm, cpu)) { + unsigned long size, flags; + + local_irq_save(flags); + start = round_down(start, PAGE_SIZE << 1); + end = round_up(end, PAGE_SIZE << 1); + size = (end - start) >> (PAGE_SHIFT + 1); + if (size <= (current_cpu_data.tlbsizestlbsets ? + current_cpu_data.tlbsize / 8 : + current_cpu_data.tlbsize / 2)) { + + int asid = cpu_asid(cpu, mm); + while (start < end) { + invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start); + start += (PAGE_SIZE << 1); + } + } else { + drop_mmu_context(mm, cpu); + } + local_irq_restore(flags); + } else { + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + } +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + unsigned long size, flags; + + local_irq_save(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + size = (size + 1) >> 1; + if (size <= (current_cpu_data.tlbsizestlbsets ? + current_cpu_data.tlbsize / 8 : + current_cpu_data.tlbsize / 2)) { + + start &= (PAGE_MASK << 1); + end += ((PAGE_SIZE << 1) - 1); + end &= (PAGE_MASK << 1); + + while (start < end) { + invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start); + start += (PAGE_SIZE << 1); + } + } else { + local_flush_tlb_kernel(); + } + local_irq_restore(flags); +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + int cpu = smp_processor_id(); + + if (asid_valid(vma->vm_mm, cpu)) { + int newpid; + + newpid = cpu_asid(cpu, vma->vm_mm); + page &= (PAGE_MASK << 1); + invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page); + } else { + cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm)); + } +} + +/* + * This one is only used for pages with the global bit set so we don't care + * much about the ASID. + */ +void local_flush_tlb_one(unsigned long page) +{ + page &= (PAGE_MASK << 1); + invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page); +} + +static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) +{ +#ifdef CONFIG_HUGETLB_PAGE + int idx; + unsigned long lo; + unsigned long flags; + + local_irq_save(flags); + + address &= (PAGE_MASK << 1); + write_csr_entryhi(address); + tlb_probe(); + idx = read_csr_tlbidx(); + write_csr_pagesize(PS_HUGE_SIZE); + lo = pmd_to_entrylo(pte_val(*ptep)); + write_csr_entrylo0(lo); + write_csr_entrylo1(lo + (HPAGE_SIZE >> 1)); + + if (idx < 0) + tlb_write_random(); + else + tlb_write_indexed(); + write_csr_pagesize(PS_DEFAULT_SIZE); + + local_irq_restore(flags); +#endif +} + +void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) +{ + int idx; + unsigned long flags; + + if (cpu_has_ptw) + return; + + /* + * Handle debugger faulting in for debugee. + */ + if (current->active_mm != vma->vm_mm) + return; + + if (pte_val(*ptep) & _PAGE_HUGE) { + __update_hugetlb(vma, address, ptep); + return; + } + + local_irq_save(flags); + + if ((unsigned long)ptep & sizeof(pte_t)) + ptep--; + + address &= (PAGE_MASK << 1); + write_csr_entryhi(address); + tlb_probe(); + idx = read_csr_tlbidx(); + write_csr_pagesize(PS_DEFAULT_SIZE); + write_csr_entrylo0(pte_val(*ptep++)); + write_csr_entrylo1(pte_val(*ptep)); + if (idx < 0) + tlb_write_random(); + else + tlb_write_indexed(); + + local_irq_restore(flags); +} + +static void setup_ptwalker(void) +{ + unsigned long pwctl0, pwctl1; + unsigned long pgd_i = 0, pgd_w = 0; + unsigned long pud_i = 0, pud_w = 0; + unsigned long pmd_i = 0, pmd_w = 0; + unsigned long pte_i = 0, pte_w = 0; + + pgd_i = PGDIR_SHIFT; + pgd_w = PAGE_SHIFT - 3; +#if CONFIG_PGTABLE_LEVELS > 3 + pud_i = PUD_SHIFT; + pud_w = PAGE_SHIFT - 3; +#endif +#if CONFIG_PGTABLE_LEVELS > 2 + pmd_i = PMD_SHIFT; + pmd_w = PAGE_SHIFT - 3; +#endif + pte_i = PAGE_SHIFT; + pte_w = PAGE_SHIFT - 3; + + pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25; + pwctl1 = pgd_i | pgd_w << 6; + + if (cpu_has_ptw) + pwctl1 |= CSR_PWCTL1_PTW; + + csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0); + csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1); + csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH); + csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL); + csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID); +} + +static void output_pgtable_bits_defines(void) +{ +#define pr_define(fmt, ...) \ + pr_debug("#define " fmt, ##__VA_ARGS__) + + pr_debug("#include \n"); + pr_debug("#include \n"); + pr_debug("\n"); + + pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); + pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); + pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); + pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); + pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); + pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); + pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); + pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); + pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); + pr_debug("\n"); +} + +#ifdef CONFIG_NUMA +static unsigned long pcpu_handlers[NR_CPUS]; +#endif +extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; + +static void setup_tlb_handler(int cpu) +{ + int i; + + setup_ptwalker(); + local_flush_tlb_all(); + + if (cpu_has_ptw) { + exception_table[EXCCODE_TLBI] = handle_tlb_load_ptw; + exception_table[EXCCODE_TLBL] = handle_tlb_load_ptw; + exception_table[EXCCODE_TLBS] = handle_tlb_store_ptw; + exception_table[EXCCODE_TLBM] = handle_tlb_modify_ptw; + } + + /* The tlb handlers are generated only once */ + if (cpu == 0) { + memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); + local_flush_icache_range(tlbrentry, tlbrentry + 0x80); + + for (i = EXCCODE_TLBL; i <= EXCCODE_TLBPE; i++) + set_handler(i * VECSIZE, exception_table[i], VECSIZE); + } else { + int vec_sz __maybe_unused; + void *addr __maybe_unused; + struct page *page __maybe_unused; + + /* Avoid lockdep warning */ + rcu_cpu_starting(cpu); + +#ifdef CONFIG_NUMA + vec_sz = sizeof(exception_handlers); + + if (pcpu_handlers[cpu]) + return; + + page = alloc_pages_node(cpu_to_node(cpu), GFP_ATOMIC, get_order(vec_sz)); + if (!page) + return; + + addr = page_address(page); + pcpu_handlers[cpu] = (unsigned long)addr; + memcpy((void *)addr, (void *)eentry, vec_sz); + local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz); + csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY); + csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY); + csr_write64(pcpu_handlers[cpu] + 80*VECSIZE, LOONGARCH_CSR_TLBRENTRY); +#endif + } +} + +void tlb_init(int cpu) +{ + write_csr_pagesize(PS_DEFAULT_SIZE); + write_csr_stlbpgsize(PS_DEFAULT_SIZE); + write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE); + + setup_tlb_handler(cpu); + output_pgtable_bits_defines(); +} diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S new file mode 100644 index 0000000000000000000000000000000000000000..528042120ab93bdbc590705621c25ffae00faadf --- /dev/null +++ b/arch/loongarch/mm/tlbex.S @@ -0,0 +1,532 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_64BIT +#include +#endif + +#define INVTLB_ADDR_GFALSE_AND_ASID 5 + +#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) + + .macro tlb_do_page_fault, write + SYM_CODE_START(tlb_do_page_fault_\write) + SAVE_ALL + csrrd a2, LOONGARCH_CSR_BADV + move a0, sp + REG_S a2, sp, PT_BVADDR + li.w a1, \write + la.abs t0, do_page_fault + jirl ra, t0, 0 + UNWIND_HINT_REGS + RESTORE_ALL_AND_RET + SYM_CODE_END(tlb_do_page_fault_\write) + .endm + + tlb_do_page_fault 0 + tlb_do_page_fault 1 + +SYM_CODE_START(handle_tlb_protect) + BACKUP_T0T1 + SAVE_ALL + move a0, sp + move a1, zero + csrrd a2, LOONGARCH_CSR_BADV + REG_S a2, sp, PT_BVADDR + la.abs t0, do_page_fault + jirl ra, t0, 0 + UNWIND_HINT_REGS + RESTORE_ALL_AND_RET +SYM_CODE_END(handle_tlb_protect) + +SYM_CODE_START(handle_tlb_load) + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 + + /* + * The vmalloc handling is not in the hotpath. + */ + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_load + csrrd t1, LOONGARCH_CSR_PGDL + +vmalloc_done_load: + /* Get PGD offset in bytes */ + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 +#if CONFIG_PGTABLE_LEVELS > 3 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 +#endif +#if CONFIG_PGTABLE_LEVELS > 2 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 +#endif + ld.d ra, t1, 0 + + /* + * For huge tlb entries, pmde doesn't contain an address but + * instead contains the tlb pte. Check the PAGE_HUGE bit and + * see if we need to jump to huge tlb processing. + */ + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_load + + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 + +#ifdef CONFIG_SMP +smp_pgtable_change_load: + ll.d t0, t1, 0 +#else + ld.d t0, t1, 0 +#endif + andi ra, t0, _PAGE_PRESENT + beqz ra, nopage_tlb_load + + ori t0, t0, _PAGE_VALID +#ifdef CONFIG_SMP + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_load +#else + st.d t0, t1, 0 +#endif + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbwr + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +#ifdef CONFIG_64BIT +vmalloc_load: + la.abs t1, swapper_pg_dir + b vmalloc_done_load +#endif + + /* This is the entry point of a huge page. */ +tlb_huge_update_load: +#ifdef CONFIG_SMP + ll.d ra, t1, 0 +#endif + andi t0, ra, _PAGE_PRESENT + beqz t0, nopage_tlb_load + +#ifdef CONFIG_SMP + ori t0, ra, _PAGE_VALID + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_load + ori t0, ra, _PAGE_VALID +#else + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, _PAGE_VALID + st.d t0, t1, 0 +#endif + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 + + /* + * A huge PTE describes an area the size of the + * configured huge page size. This is twice the + * of the large TLB entry size we intend to use. + * A TLB entry half the size of the configured + * huge page size is configured into entrylo0 + * and entrylo1 to cover the contiguous huge PTE + * address space. + */ + /* Huge page: Move Global bit */ + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 + + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 + + /* Convert to entrylo1 */ + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 + + /* Set huge page tlb entry size */ + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + tlbfill + + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +nopage_tlb_load: + dbar 0x700 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_0 + jr t0 +SYM_CODE_END(handle_tlb_load) + +SYM_CODE_START(handle_tlb_load_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la.abs t0, tlb_do_page_fault_0 + jirl zero, t0, 0 +SYM_CODE_END(handle_tlb_load_ptw) + +SYM_CODE_START(handle_tlb_store) + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 + + /* + * The vmalloc handling is not in the hotpath. + */ + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_store + csrrd t1, LOONGARCH_CSR_PGDL + +vmalloc_done_store: + /* Get PGD offset in bytes */ + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 +#if CONFIG_PGTABLE_LEVELS > 3 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 +#endif +#if CONFIG_PGTABLE_LEVELS > 2 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 +#endif + ld.d ra, t1, 0 + + /* + * For huge tlb entries, pmde doesn't contain an address but + * instead contains the tlb pte. Check the PAGE_HUGE bit and + * see if we need to jump to huge tlb processing. + */ + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_store + + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 + +#ifdef CONFIG_SMP +smp_pgtable_change_store: + ll.d t0, t1, 0 +#else + ld.d t0, t1, 0 +#endif + andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE + xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE + bnez ra, nopage_tlb_store + + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) +#ifdef CONFIG_SMP + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_store +#else + st.d t0, t1, 0 +#endif + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbwr + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +#ifdef CONFIG_64BIT +vmalloc_store: + la.abs t1, swapper_pg_dir + b vmalloc_done_store +#endif + + /* This is the entry point of a huge page. */ +tlb_huge_update_store: +#ifdef CONFIG_SMP + ll.d ra, t1, 0 +#endif + andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE + xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE + bnez t0, nopage_tlb_store + +#ifdef CONFIG_SMP + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_store + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) +#else + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + st.d t0, t1, 0 +#endif + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 + + /* + * A huge PTE describes an area the size of the + * configured huge page size. This is twice the + * of the large TLB entry size we intend to use. + * A TLB entry half the size of the configured + * huge page size is configured into entrylo0 + * and entrylo1 to cover the contiguous huge PTE + * address space. + */ + /* Huge page: Move Global bit */ + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 + + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 + + /* Convert to entrylo1 */ + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 + + /* Set huge page tlb entry size */ + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + tlbfill + + /* Reset default page size */ + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +nopage_tlb_store: + dbar 0x700 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_1 + jr t0 +SYM_CODE_END(handle_tlb_store) + +SYM_CODE_START(handle_tlb_store_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la.abs t0, tlb_do_page_fault_1 + jirl zero, t0, 0 +SYM_CODE_END(handle_tlb_store_ptw) + +SYM_CODE_START(handle_tlb_modify) + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 + + /* + * The vmalloc handling is not in the hotpath. + */ + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_modify + csrrd t1, LOONGARCH_CSR_PGDL + +vmalloc_done_modify: + /* Get PGD offset in bytes */ + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 +#if CONFIG_PGTABLE_LEVELS > 3 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 +#endif +#if CONFIG_PGTABLE_LEVELS > 2 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 +#endif + ld.d ra, t1, 0 + + /* + * For huge tlb entries, pmde doesn't contain an address but + * instead contains the tlb pte. Check the PAGE_HUGE bit and + * see if we need to jump to huge tlb processing. + */ + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_modify + + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 + +#ifdef CONFIG_SMP +smp_pgtable_change_modify: + ll.d t0, t1, 0 +#else + ld.d t0, t1, 0 +#endif + andi ra, t0, _PAGE_WRITE + beqz ra, nopage_tlb_modify + + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) +#ifdef CONFIG_SMP + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_modify +#else + st.d t0, t1, 0 +#endif + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbwr + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +#ifdef CONFIG_64BIT +vmalloc_modify: + la.abs t1, swapper_pg_dir + b vmalloc_done_modify +#endif + + /* This is the entry point of a huge page. */ +tlb_huge_update_modify: +#ifdef CONFIG_SMP + ll.d ra, t1, 0 +#endif + andi t0, ra, _PAGE_WRITE + beqz t0, nopage_tlb_modify + +#ifdef CONFIG_SMP + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_modify + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) +#else + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + st.d t0, t1, 0 +#endif + csrrd ra, LOONGARCH_CSR_ASID + csrrd t1, LOONGARCH_CSR_BADV + andi ra, ra, CSR_ASID_ASID + invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 + + /* + * A huge PTE describes an area the size of the + * configured huge page size. This is twice the + * of the large TLB entry size we intend to use. + * A TLB entry half the size of the configured + * huge page size is configured into entrylo0 + * and entrylo1 to cover the contiguous huge PTE + * address space. + */ + /* Huge page: Move Global bit */ + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 + + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 + + /* Convert to entrylo1 */ + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 + + /* Set huge page tlb entry size */ + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + tlbfill + + /* Reset default page size */ + addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) + addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + +nopage_tlb_modify: + dbar 0x700 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_1 + jr t0 +SYM_CODE_END(handle_tlb_modify) + +SYM_CODE_START(handle_tlb_modify_ptw) + csrwr t0, LOONGARCH_CSR_KS0 + csrwr t1, LOONGARCH_CSR_KS1 + la.abs t0, tlb_do_page_fault_1 + jirl zero, t0, 0 +SYM_CODE_END(handle_tlb_modify_ptw) + +SYM_CODE_START(handle_tlb_refill) + csrwr t0, LOONGARCH_CSR_TLBRSAVE + csrrd t0, LOONGARCH_CSR_PGD + lddir t0, t0, 3 +#if CONFIG_PGTABLE_LEVELS > 3 + lddir t0, t0, 2 +#endif +#if CONFIG_PGTABLE_LEVELS > 2 + lddir t0, t0, 1 +#endif + ldpte t0, 0 + ldpte t0, 1 + tlbfill + csrrd t0, LOONGARCH_CSR_TLBRSAVE + ertn +SYM_CODE_END(handle_tlb_refill) diff --git a/arch/loongarch/net/Makefile b/arch/loongarch/net/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..42fa082caa93f57dd8b3ea05d9525cbef1a574cd --- /dev/null +++ b/arch/loongarch/net/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for arch/loongarch/net +# +# Copyright (C) 2021 Loongson Technology Corporation Limited +# +obj-$(CONFIG_BPF_JIT) += ebpf_jit.o diff --git a/arch/loongarch/net/ebpf_jit.c b/arch/loongarch/net/ebpf_jit.c new file mode 100644 index 0000000000000000000000000000000000000000..b62f3f048c132b5882922fd271626bc553b542f4 --- /dev/null +++ b/arch/loongarch/net/ebpf_jit.c @@ -0,0 +1,978 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * BPF JIT compiler for LoongArch + * + * Copyright (C) 2021 Loongson Technology Corporation Limited + */ +#include "ebpf_jit.h" + +#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) +#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define TMP_REG_3 (MAX_BPF_JIT_REG + 2) +#define REG_TCC (MAX_BPF_JIT_REG + 3) +#define TCC_SAVED (MAX_BPF_JIT_REG + 4) + +#define SAVE_RA BIT(0) +#define SAVE_TCC BIT(1) + +static const int regmap[] = { + /* return value from in-kernel function, and exit value for eBPF program */ + [BPF_REG_0] = LOONGARCH_GPR_A5, + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = LOONGARCH_GPR_A0, + [BPF_REG_2] = LOONGARCH_GPR_A1, + [BPF_REG_3] = LOONGARCH_GPR_A2, + [BPF_REG_4] = LOONGARCH_GPR_A3, + [BPF_REG_5] = LOONGARCH_GPR_A4, + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = LOONGARCH_GPR_S0, + [BPF_REG_7] = LOONGARCH_GPR_S1, + [BPF_REG_8] = LOONGARCH_GPR_S2, + [BPF_REG_9] = LOONGARCH_GPR_S3, + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = LOONGARCH_GPR_S4, + /* temporary register for blinding constants */ + [BPF_REG_AX] = LOONGARCH_GPR_T0, + /* temporary register for internal BPF JIT */ + [TMP_REG_1] = LOONGARCH_GPR_T1, + [TMP_REG_2] = LOONGARCH_GPR_T2, + [TMP_REG_3] = LOONGARCH_GPR_T3, + /* tail call */ + [REG_TCC] = LOONGARCH_GPR_A6, + /* store A6 in S5 if program do calls */ + [TCC_SAVED] = LOONGARCH_GPR_S5, +}; + +static void mark_call(struct jit_ctx *ctx) +{ + ctx->flags |= SAVE_RA; +} + +static void mark_tail_call(struct jit_ctx *ctx) +{ + ctx->flags |= SAVE_TCC; +} + +static bool seen_call(struct jit_ctx *ctx) +{ + return (ctx->flags & SAVE_RA); +} + +static bool seen_tail_call(struct jit_ctx *ctx) +{ + return (ctx->flags & SAVE_TCC); +} + +static u8 tail_call_reg(struct jit_ctx *ctx) +{ + if (seen_call(ctx)) + return regmap[TCC_SAVED]; + + return regmap[REG_TCC]; +} + +/* + * eBPF prog stack layout: + * + * high + * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP + * | $ra | + * +-------------------------+ + * | $fp | + * +-------------------------+ + * | $s0 | + * +-------------------------+ + * | $s1 | + * +-------------------------+ + * | $s2 | + * +-------------------------+ + * | $s3 | + * +-------------------------+ + * | $s4 | + * +-------------------------+ + * | $s5 | + * +-------------------------+ <--BPF_REG_FP + * | prog->aux->stack_depth | + * | (optional) | + * current $sp -------------> +-------------------------+ + * low + */ +static void build_prologue(struct jit_ctx *ctx) +{ + int stack_adjust = 0, store_offset, bpf_stack_adjust; + + bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + + stack_adjust += sizeof(long); /* LOONGARCH_GPR_RA */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_FP */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S0 */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S1 */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S2 */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S3 */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S4 */ + stack_adjust += sizeof(long); /* LOONGARCH_GPR_S5 */ + + stack_adjust = round_up(stack_adjust, 16); + stack_adjust += bpf_stack_adjust; + + /* + * First instruction initializes the tail call count (TCC). + * On tail call we skip this instruction, and the TCC is + * passed in REG_TCC from the caller. + */ + emit_insn(ctx, addid, regmap[REG_TCC], LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); + + store_offset = stack_adjust - sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); + + if (bpf_stack_adjust) + emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); + + /* + * Program contains calls and tail calls, so REG_TCC need + * to be saved across calls. + */ + if (seen_tail_call(ctx) && seen_call(ctx)) + move_reg(ctx, regmap[TCC_SAVED], regmap[REG_TCC]); + + ctx->stack_size = stack_adjust; +} + +static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) +{ + int stack_adjust = ctx->stack_size; + int load_offset; + + load_offset = stack_adjust - sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); + + if (!is_tail_call) { + /* Set return value */ + move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]); + /* Return to the caller */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); + } else { + /* + * Call the next bpf prog and skip the first instruction + * of TCC initialization. + */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, regmap[TMP_REG_3], 1); + } +} + +void build_epilogue(struct jit_ctx *ctx) +{ + __build_epilogue(ctx, false); +} + +/* initialized on the first pass of build_body() */ +static int out_offset = -1; +static int emit_bpf_tail_call(struct jit_ctx *ctx) +{ + int off; + u8 tcc = tail_call_reg(ctx); + u8 a1 = LOONGARCH_GPR_A1; + u8 a2 = LOONGARCH_GPR_A2; + u8 tmp1 = regmap[TMP_REG_1]; + u8 tmp2 = regmap[TMP_REG_2]; + u8 tmp3 = regmap[TMP_REG_3]; + const int idx0 = ctx->idx; + +#define cur_offset (ctx->idx - idx0) +#define jmp_offset (out_offset - (cur_offset)) + + /* + * a0: &ctx + * a1: &array + * a2: index + * + * if (index >= array->map.max_entries) + * goto out; + */ + off = offsetof(struct bpf_array, map.max_entries); + emit_insn(ctx, ldwu, tmp1, a1, off); + /* bgeu $a2, $t1, jmp_offset */ + emit_tailcall_jump(ctx, BPF_JGE, a2, tmp1, jmp_offset); + + /* + * if (TCC-- < 0) + * goto out; + */ + emit_insn(ctx, addid, tmp1, tcc, -1); + emit_tailcall_jump(ctx, BPF_JSLT, tcc, LOONGARCH_GPR_ZERO, jmp_offset); + + /* + * prog = array->ptrs[index]; + * if (!prog) + * goto out; + */ + emit_insn(ctx, sllid, tmp2, a2, 3); + emit_insn(ctx, addd, tmp2, tmp2, a1); + off = offsetof(struct bpf_array, ptrs); + emit_insn(ctx, ldd, tmp2, tmp2, off); + /* beq $t2, $zero, jmp_offset */ + emit_tailcall_jump(ctx, BPF_JEQ, tmp2, LOONGARCH_GPR_ZERO, jmp_offset); + + /* goto *(prog->bpf_func + 4); */ + off = offsetof(struct bpf_prog, bpf_func); + emit_insn(ctx, ldd, tmp3, tmp2, off); + move_reg(ctx, tcc, tmp1); + __build_epilogue(ctx, true); + + /* out: */ + if (out_offset == -1) + out_offset = cur_offset; + if (cur_offset != out_offset) { + pr_err_once("tail_call out_offset = %d, expected %d!\n", + cur_offset, out_offset); + return -1; + } + + return 0; +#undef cur_offset +#undef jmp_offset +} + +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) +{ + bool is32 = (BPF_CLASS(insn->code) == BPF_ALU); + const u8 code = insn->code; + const u8 cond = BPF_OP(code); + const u8 dst = regmap[insn->dst_reg]; + const u8 src = regmap[insn->src_reg]; + const u8 tmp = regmap[TMP_REG_1]; + const u8 tmp2 = regmap[TMP_REG_2]; + const s16 off = insn->off; + const s32 imm = insn->imm; + int i = insn - ctx->prog->insnsi; + int jmp_offset; + bool func_addr_fixed; + u64 func_addr; + u64 imm64; + int ret; + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + case BPF_ALU64 | BPF_MOV | BPF_X: + move_reg(ctx, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + case BPF_ALU64 | BPF_MOV | BPF_K: + move_imm32(ctx, dst, imm, is32); + break; + + /* dst = dst + src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit_insn(ctx, addd, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst + imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + if (is_signed_imm12(imm)) { + emit_insn(ctx, addid, dst, dst, imm); + } else { + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, addd, dst, dst, tmp); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst - src */ + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit_insn(ctx, subd, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst - imm */ + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + if (is_signed_imm12(-imm)) { + emit_insn(ctx, addid, dst, dst, -imm); + } else { + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, subd, dst, dst, tmp); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst * src */ + case BPF_ALU | BPF_MUL | BPF_X: + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_insn(ctx, muld, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst * imm */ + case BPF_ALU | BPF_MUL | BPF_K: + case BPF_ALU64 | BPF_MUL | BPF_K: + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, muld, dst, dst, tmp); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst / src */ + case BPF_ALU | BPF_DIV | BPF_X: + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_insn(ctx, divdu, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst / imm */ + case BPF_ALU | BPF_DIV | BPF_K: + case BPF_ALU64 | BPF_DIV | BPF_K: + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, divdu, dst, dst, tmp); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst % src */ + case BPF_ALU | BPF_MOD | BPF_X: + case BPF_ALU64 | BPF_MOD | BPF_X: + emit_insn(ctx, moddu, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst % imm */ + case BPF_ALU | BPF_MOD | BPF_K: + case BPF_ALU64 | BPF_MOD | BPF_K: + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, moddu, dst, dst, tmp); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst & src */ + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit_insn(ctx, and, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst & imm */ + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, andi, dst, dst, imm); + } else { + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, and, dst, dst, tmp); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst | src */ + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit_insn(ctx, or, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst | imm */ + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, ori, dst, dst, imm); + } else { + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, or, dst, dst, tmp); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst ^ src */ + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit_insn(ctx, xor, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + /* dst = dst ^ imm */ + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, xori, dst, dst, imm); + } else { + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, xor, dst, dst, tmp); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst << src (logical) */ + case BPF_ALU | BPF_LSH | BPF_X: + emit_insn(ctx, sllw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_insn(ctx, slld, dst, dst, src); + break; + /* dst = dst << imm (logical) */ + case BPF_ALU | BPF_LSH | BPF_K: + emit_insn(ctx, slliw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_LSH | BPF_K: + emit_insn(ctx, sllid, dst, dst, imm); + break; + + /* dst = dst >> src (logical) */ + case BPF_ALU | BPF_RSH | BPF_X: + emit_insn(ctx, srlw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_RSH | BPF_X: + emit_insn(ctx, srld, dst, dst, src); + break; + /* dst = dst >> imm (logical) */ + case BPF_ALU | BPF_RSH | BPF_K: + emit_insn(ctx, srliw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_RSH | BPF_K: + emit_insn(ctx, srlid, dst, dst, imm); + break; + + /* dst = dst >> src (arithmetic) */ + case BPF_ALU | BPF_ARSH | BPF_X: + emit_insn(ctx, sraw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_insn(ctx, srad, dst, dst, src); + break; + /* dst = dst >> imm (arithmetic) */ + case BPF_ALU | BPF_ARSH | BPF_K: + emit_insn(ctx, sraiw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_insn(ctx, sraid, dst, dst, imm); + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_LE: + switch (imm) { + case 16: + /* zero-extend 16 bits into 64 bits */ + emit_insn(ctx, sllid, dst, dst, 48); + emit_insn(ctx, srlid, dst, dst, 48); + break; + case 32: + /* zero-extend 32 bits into 64 bits */ + emit_zext_32(ctx, dst, is32); + break; + case 64: + /* do nothing */ + break; + } + break; + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + emit_insn(ctx, revb2h, dst, dst); + /* zero-extend 16 bits into 64 bits */ + emit_insn(ctx, sllid, dst, dst, 48); + emit_insn(ctx, srlid, dst, dst, 48); + break; + case 32: + emit_insn(ctx, revb2w, dst, dst); + /* zero-extend 32 bits into 64 bits */ + emit_zext_32(ctx, dst, is32); + break; + case 64: + emit_insn(ctx, revbd, dst, dst); + break; + } + break; + + /* PC += off if dst cond src */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + jmp_offset = bpf2la_offset(i, off, ctx); + emit_cond_jump(ctx, cond, dst, src, jmp_offset); + break; + + /* PC += off if dst cond imm */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + jmp_offset = bpf2la_offset(i, off, ctx); + move_imm32(ctx, tmp, imm, is32); + emit_cond_jump(ctx, cond, dst, tmp, jmp_offset); + break; + + /* PC += off if dst & src */ + case BPF_JMP | BPF_JSET | BPF_X: + jmp_offset = bpf2la_offset(i, off, ctx); + emit_insn(ctx, and, tmp, dst, src); + emit_cond_jump(ctx, cond, tmp, LOONGARCH_GPR_ZERO, jmp_offset); + break; + /* PC += off if dst & imm */ + case BPF_JMP | BPF_JSET | BPF_K: + jmp_offset = bpf2la_offset(i, off, ctx); + move_imm32(ctx, tmp, imm, is32); + emit_insn(ctx, and, tmp, dst, tmp); + emit_cond_jump(ctx, cond, tmp, LOONGARCH_GPR_ZERO, jmp_offset); + break; + + /* PC += off */ + case BPF_JMP | BPF_JA: + jmp_offset = bpf2la_offset(i, off, ctx); + emit_uncond_jump(ctx, jmp_offset, false); + break; + + /* function call */ + case BPF_JMP | BPF_CALL: + mark_call(ctx); + ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, + &func_addr, &func_addr_fixed); + if (ret < 0) + return ret; + + move_imm64(ctx, tmp, func_addr, is32); + emit_insn(ctx, jirl, LOONGARCH_GPR_RA, tmp, 0); + move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); + break; + + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + mark_tail_call(ctx); + if (emit_bpf_tail_call(ctx)) + return -EINVAL; + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + emit_sext_32(ctx, regmap[BPF_REG_0]); + /* + * Optimization: when last instruction is EXIT, + * simply fallthrough to epilogue. + */ + if (i == ctx->prog->len - 1) + break; + + jmp_offset = epilogue_offset(ctx); + emit_uncond_jump(ctx, jmp_offset, true); + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; + move_imm64(ctx, dst, imm64, is32); + return 1; + + /* dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: + if (is_signed_imm12(off)) { + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, ldbu, dst, src, off); + break; + case BPF_H: + emit_insn(ctx, ldhu, dst, src, off); + break; + case BPF_W: + emit_insn(ctx, ldwu, dst, src, off); + break; + case BPF_DW: + emit_insn(ctx, ldd, dst, src, off); + break; + } + } else { + move_imm32(ctx, tmp, off, is32); + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, ldxbu, dst, src, tmp); + break; + case BPF_H: + emit_insn(ctx, ldxhu, dst, src, tmp); + break; + case BPF_W: + emit_insn(ctx, ldxwu, dst, src, tmp); + break; + case BPF_DW: + emit_insn(ctx, ldxd, dst, src, tmp); + break; + } + } + break; + + /* *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_DW: + move_imm32(ctx, tmp, imm, is32); + if (is_signed_imm12(off)) { + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, stb, tmp, dst, off); + break; + case BPF_H: + emit_insn(ctx, sth, tmp, dst, off); + break; + case BPF_W: + emit_insn(ctx, stw, tmp, dst, off); + break; + case BPF_DW: + emit_insn(ctx, std, tmp, dst, off); + break; + } + } else { + move_imm32(ctx, tmp2, off, is32); + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, stxb, tmp, dst, tmp2); + break; + case BPF_H: + emit_insn(ctx, stxh, tmp, dst, tmp2); + break; + case BPF_W: + emit_insn(ctx, stxw, tmp, dst, tmp2); + break; + case BPF_DW: + emit_insn(ctx, stxd, tmp, dst, tmp2); + break; + } + } + break; + + /* *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_DW: + if (is_signed_imm12(off)) { + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, stb, src, dst, off); + break; + case BPF_H: + emit_insn(ctx, sth, src, dst, off); + break; + case BPF_W: + emit_insn(ctx, stw, src, dst, off); + break; + case BPF_DW: + emit_insn(ctx, std, src, dst, off); + break; + } + } else { + move_imm32(ctx, tmp, off, is32); + switch (BPF_SIZE(code)) { + case BPF_B: + emit_insn(ctx, stxb, src, dst, tmp); + break; + case BPF_H: + emit_insn(ctx, stxh, src, dst, tmp); + break; + case BPF_W: + emit_insn(ctx, stxw, src, dst, tmp); + break; + case BPF_DW: + emit_insn(ctx, stxd, src, dst, tmp); + break; + } + } + break; + + /* atomic_add: lock *(size *)(dst + off) += src */ + case BPF_STX | BPF_XADD | BPF_W: + case BPF_STX | BPF_XADD | BPF_DW: + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op code %02x\n", insn->imm); + return -EINVAL; + } + + move_imm32(ctx, tmp, off, is32); + emit_insn(ctx, addd, tmp, dst, tmp); + switch (BPF_SIZE(insn->code)) { + case BPF_W: + emit_insn(ctx, amaddw, tmp2, src, tmp); + break; + case BPF_DW: + emit_insn(ctx, amaddd, tmp2, src, tmp); + break; + } + break; + + default: + pr_err("bpf_jit: unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +static int build_body(struct jit_ctx *ctx, bool extra_pass) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + if (!ctx->image) + ctx->offset[i] = ctx->idx; + + ret = build_insn(insn, ctx, extra_pass); + if (ret > 0) { + i++; + if (!ctx->image) + ctx->offset[i] = ctx->idx; + continue; + } + if (ret) + return ret; + } + + if (!ctx->image) + ctx->offset[i] = ctx->idx; + + return 0; +} + +static inline void bpf_flush_icache(void *start, void *end) +{ + flush_icache_range((unsigned long)start, (unsigned long)end); +} + +/* Fill space with illegal instructions */ +static void jit_fill_hole(void *area, unsigned int size) +{ + u32 *ptr; + + /* We are guaranteed to have aligned memory */ + for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) + *ptr++ = INSN_BREAK; +} + +static int validate_code(struct jit_ctx *ctx) +{ + int i; + union loongarch_instruction insn; + + for (i = 0; i < ctx->idx; i++) { + insn = ctx->image[i]; + /* Check INSN_BREAK */ + if (insn.word == INSN_BREAK) + return -1; + } + + return 0; +} + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + struct bpf_prog *tmp, *orig_prog = prog; + struct bpf_binary_header *header; + struct jit_data *jit_data; + struct jit_ctx ctx; + bool tmp_blinded = false; + bool extra_pass = false; + int image_size; + u8 *image_ptr; + + /* + * If BPF JIT was not enabled then we must fall back to + * the interpreter. + */ + if (!prog->jit_requested) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + /* + * If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. Otherwise, we save + * the new JITed code. + */ + if (IS_ERR(tmp)) + return orig_prog; + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + if (!jit_data) { + prog = orig_prog; + goto out; + } + prog->aux->jit_data = jit_data; + } + if (jit_data->ctx.offset) { + ctx = jit_data->ctx; + image_ptr = jit_data->image; + header = jit_data->header; + extra_pass = true; + image_size = sizeof(u32) * ctx.idx; + goto skip_init_ctx; + } + + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.offset = kcalloc(prog->len + 1, sizeof(*ctx.offset), GFP_KERNEL); + if (!ctx.offset) { + prog = orig_prog; + goto out_off; + } + + /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ + if (build_body(&ctx, extra_pass)) { + prog = orig_prog; + goto out_off; + } + build_prologue(&ctx); + ctx.epilogue_offset = ctx.idx; + build_epilogue(&ctx); + + /* Now we know the actual image size. + * As each LoongArch instruction is of length 32bit, + * we are translating number of JITed intructions into + * the size required to store these JITed code. + */ + image_size = sizeof(u32) * ctx.idx; + /* Now we know the size of the structure to make */ + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (!header) { + prog = orig_prog; + goto out_off; + } + + /* 2. Now, the actual pass to generate final JIT code */ + ctx.image = (union loongarch_instruction *)image_ptr; +skip_init_ctx: + ctx.idx = 0; + + build_prologue(&ctx); + if (build_body(&ctx, extra_pass)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + build_epilogue(&ctx); + + /* 3. Extra pass to validate JITed code */ + if (validate_code(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + /* And we're done */ + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, image_size, 2, ctx.image); + + /* Update the icache */ + bpf_flush_icache(header, ctx.image + ctx.idx); + + if (!prog->is_func || extra_pass) { + if (extra_pass && ctx.idx != jit_data->ctx.idx) { + pr_err_once("multi-func JIT bug %d != %d\n", + ctx.idx, jit_data->ctx.idx); + bpf_jit_binary_free(header); + prog->bpf_func = NULL; + prog->jited = 0; + goto out_off; + } + bpf_jit_binary_lock_ro(header); + } else { + jit_data->ctx = ctx; + jit_data->image = image_ptr; + jit_data->header = header; + } + prog->bpf_func = (void *)ctx.image; + prog->jited = 1; + prog->jited_len = image_size; + + if (!prog->is_func || extra_pass) { +out_off: + kfree(ctx.offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } +out: + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? + tmp : orig_prog); + + out_offset = -1; + return prog; +} diff --git a/arch/loongarch/net/ebpf_jit.h b/arch/loongarch/net/ebpf_jit.h new file mode 100644 index 0000000000000000000000000000000000000000..8c39316d3743712912fd211dd2397b785800d16b --- /dev/null +++ b/arch/loongarch/net/ebpf_jit.h @@ -0,0 +1,842 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * BPF JIT compiler for LoongArch + * + * Copyright (C) 2021 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +struct jit_ctx { + const struct bpf_prog *prog; + unsigned int idx; + unsigned int flags; + unsigned int epilogue_offset; + u32 *offset; + union loongarch_instruction *image; + u32 stack_size; +}; + +struct jit_data { + struct bpf_binary_header *header; + u8 *image; + struct jit_ctx ctx; +}; + +#define emit_insn(ctx, func, ...) \ +do { \ + if (ctx->image != NULL) { \ + union loongarch_instruction *insn = &ctx->image[ctx->idx]; \ + emit_##func(insn, ##__VA_ARGS__); \ + } \ + ctx->idx++; \ +} while (0) + +static inline bool is_unsigned_imm(unsigned long val, unsigned int bit) +{ + return val < (1UL << bit); +} + +static inline bool is_signed_imm(long val, unsigned int bit) +{ + return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1)); +} + +#define is_signed_imm12(val) is_signed_imm(val, 12) +#define is_signed_imm16(val) is_signed_imm(val, 16) +#define is_signed_imm26(val) is_signed_imm(val, 26) +#define is_signed_imm32(val) is_signed_imm(val, 32) +#define is_signed_imm52(val) is_signed_imm(val, 52) +#define is_unsigned_imm12(val) is_unsigned_imm(val, 12) +#define is_unsigned_imm32(val) is_unsigned_imm(val, 32) + +static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx) +{ + /* BPF JMP offset is relative to the next instruction */ + bpf_insn++; + /* + * Whereas la64 branch instructions encode the offset + * from the branch itself, so we must subtract 1 from the + * instruction offset. + */ + return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1)); +} + +static inline int epilogue_offset(const struct jit_ctx *ctx) +{ + int to = ctx->epilogue_offset; + int from = ctx->idx; + + return (to - from); +} + +static inline void emit_ldbu(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = ldbu_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_ldhu(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = ldhu_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_ldwu(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = ldwu_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_ldd(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = ldd_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_stb(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = stb_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_sth(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = sth_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_stw(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = stw_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_std(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = std_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_ldxbu(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = ldxbu_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_ldxhu(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = ldxhu_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_ldxwu(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = ldxwu_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_ldxd(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = ldxd_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_stxb(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = stxb_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_stxh(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = stxh_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_stxw(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = stxw_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_stxd(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = stxd_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_amaddw(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rk, enum loongarch_gpr rj) +{ + insn->reg3_format.opcode = amaddw_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rk = rk; + insn->reg3_format.rj = rj; +} + +static inline void emit_amaddd(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rk, enum loongarch_gpr rj) +{ + insn->reg3_format.opcode = amaddd_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rk = rk; + insn->reg3_format.rj = rj; +} + +static inline void emit_addd(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = addd_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_addiw(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = addiw_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_addid(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = addid_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_subd(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = subd_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_muld(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = muld_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_divdu(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = divdu_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_moddu(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = moddu_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_and(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = and_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_andi(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui12_format.opcode = andi_op; + insn->reg2ui12_format.simmediate = imm; + insn->reg2ui12_format.rd = rd; + insn->reg2ui12_format.rj = rj; +} + +static inline void emit_or(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = or_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_ori(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui12_format.opcode = ori_op; + insn->reg2ui12_format.simmediate = imm; + insn->reg2ui12_format.rd = rd; + insn->reg2ui12_format.rj = rj; +} + +static inline void emit_xor(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = xor_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_xori(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui12_format.opcode = xori_op; + insn->reg2ui12_format.simmediate = imm; + insn->reg2ui12_format.rd = rd; + insn->reg2ui12_format.rj = rj; +} + +static inline void emit_lu12iw(union loongarch_instruction *insn, + enum loongarch_gpr rd, int imm) +{ + insn->reg1i20_format.opcode = lu12iw_op; + insn->reg1i20_format.simmediate = imm; + insn->reg1i20_format.rd = rd; +} + +static inline void emit_lu32id(union loongarch_instruction *insn, + enum loongarch_gpr rd, int imm) +{ + insn->reg1i20_format.opcode = lu32id_op; + insn->reg1i20_format.simmediate = imm; + insn->reg1i20_format.rd = rd; +} + +static inline void emit_lu52id(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + insn->reg2i12_format.opcode = lu52id_op; + insn->reg2i12_format.simmediate = imm; + insn->reg2i12_format.rd = rd; + insn->reg2i12_format.rj = rj; +} + +static inline void emit_sllw(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = sllw_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_slliw(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui5_format.opcode = slliw_op; + insn->reg2ui5_format.simmediate = imm; + insn->reg2ui5_format.rd = rd; + insn->reg2ui5_format.rj = rj; +} + +static inline void emit_slld(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = slld_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_sllid(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui6_format.opcode = sllid_op; + insn->reg2ui6_format.simmediate = imm; + insn->reg2ui6_format.rd = rd; + insn->reg2ui6_format.rj = rj; +} + +static inline void emit_srlw(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = srlw_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_srliw(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui5_format.opcode = srliw_op; + insn->reg2ui5_format.simmediate = imm; + insn->reg2ui5_format.rd = rd; + insn->reg2ui5_format.rj = rj; +} + +static inline void emit_srld(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = srld_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_srlid(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui6_format.opcode = srlid_op; + insn->reg2ui6_format.simmediate = imm; + insn->reg2ui6_format.rd = rd; + insn->reg2ui6_format.rj = rj; +} + +static inline void emit_sraw(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = sraw_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_sraiw(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui5_format.opcode = sraid_op; + insn->reg2ui5_format.simmediate = imm; + insn->reg2ui5_format.rd = rd; + insn->reg2ui5_format.rj = rj; +} + +static inline void emit_srad(union loongarch_instruction *insn, enum loongarch_gpr rd, + enum loongarch_gpr rj, enum loongarch_gpr rk) +{ + insn->reg3_format.opcode = srad_op; + insn->reg3_format.rd = rd; + insn->reg3_format.rj = rj; + insn->reg3_format.rk = rk; +} + +static inline void emit_sraid(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm) +{ + insn->reg2ui6_format.opcode = sraid_op; + insn->reg2ui6_format.simmediate = imm; + insn->reg2ui6_format.rd = rd; + insn->reg2ui6_format.rj = rj; +} + +static inline void emit_beq(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = beq_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_bne(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = bne_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_blt(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = blt_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_bge(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = bge_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_bltu(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = bltu_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_bgeu(union loongarch_instruction *insn, + enum loongarch_gpr rj, enum loongarch_gpr rd, int offset) +{ + insn->reg2i16_format.opcode = bgeu_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rj = rj; + insn->reg2i16_format.rd = rd; +} + +static inline void emit_b(union loongarch_instruction *insn, int offset) +{ + unsigned int simmediate_l, simmediate_h; + + simmediate_l = offset & 0xffff; + offset >>= 16; + simmediate_h = offset & 0x3ff; + + insn->reg0i26_format.opcode = b_op; + insn->reg0i26_format.simmediate_l = simmediate_l; + insn->reg0i26_format.simmediate_h = simmediate_h; +} + +static inline void emit_jirl(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj, int offset) +{ + insn->reg2i16_format.opcode = jirl_op; + insn->reg2i16_format.simmediate = offset; + insn->reg2i16_format.rd = rd; + insn->reg2i16_format.rj = rj; +} + +static inline void emit_pcaddu18i(union loongarch_instruction *insn, + enum loongarch_gpr rd, int imm) +{ + insn->reg1i20_format.opcode = pcaddu18i_op; + insn->reg1i20_format.simmediate = imm; + insn->reg1i20_format.rd = rd; +} + +static inline void emit_revb2h(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj) +{ + insn->reg2_format.opcode = revb2h_op; + insn->reg2_format.rd = rd; + insn->reg2_format.rj = rj; +} + +static inline void emit_revb2w(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj) +{ + insn->reg2_format.opcode = revb2w_op; + insn->reg2_format.rd = rd; + insn->reg2_format.rj = rj; +} + +static inline void emit_revbd(union loongarch_instruction *insn, + enum loongarch_gpr rd, enum loongarch_gpr rj) +{ + insn->reg2_format.opcode = revbd_op; + insn->reg2_format.rd = rd; + insn->reg2_format.rj = rj; +} + +/* Zero-extend 32 bits into 64 bits */ +static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) +{ + if (!is32) + return; + + /* Clear the upper 32 bits */ + emit_insn(ctx, lu32id, reg, 0); +} + +/* Signed-extend 32 bits into 64 bits */ +static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg) +{ + emit_insn(ctx, addiw, reg, reg, 0); +} + +static inline void move_imm32(struct jit_ctx *ctx, enum loongarch_gpr rd, + int imm32, bool is32) +{ + int si20; + u32 ui12; + + /* or rd, $zero, $zero */ + if (imm32 == 0) { + emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO); + return; + } + + /* addiw rd, $zero, imm_11_0(signed) */ + if (is_signed_imm12(imm32)) { + emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm32); + goto zext; + } + + /* ori rd, $zero, imm_11_0(unsigned) */ + if (is_unsigned_imm12(imm32)) { + emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm32); + goto zext; + } + + /* lu12iw rd, imm_31_12(signed) */ + si20 = (imm32 >> 12) & 0xfffff; + emit_insn(ctx, lu12iw, rd, si20); + + /* ori rd, rd, imm_11_0(unsigned) */ + ui12 = imm32 & 0xfff; + if (ui12 != 0) + emit_insn(ctx, ori, rd, rd, ui12); + +zext: + emit_zext_32(ctx, rd, is32); +} + +static inline void move_imm64(struct jit_ctx *ctx, enum loongarch_gpr rd, + long imm64, bool is32) +{ + int imm32, si20, si12; + long imm52; + + si12 = (imm64 >> 52) & 0xfff; + imm52 = imm64 & 0xfffffffffffff; + /* lu52id rd, $zero, imm_63_52(signed) */ + if (si12 != 0 && imm52 == 0) { + emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, si12); + return; + } + + imm32 = imm64 & 0xffffffff; + move_imm32(ctx, rd, imm32, is32); + + if (!is_signed_imm32(imm64)) { + if (imm52 != 0) { + /* lu32id rd, imm_51_32(signed) */ + si20 = (imm64 >> 32) & 0xfffff; + emit_insn(ctx, lu32id, rd, si20); + } + + /* lu52id rd, rd, imm_63_52(signed) */ + if (!is_signed_imm52(imm64)) + emit_insn(ctx, lu52id, rd, rd, si12); + } +} + +static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd, + enum loongarch_gpr rj) +{ + emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO); +} + +static inline int invert_jump_cond(u8 cond) +{ + switch (cond) { + case BPF_JEQ: + return BPF_JNE; + case BPF_JNE: + case BPF_JSET: + return BPF_JEQ; + case BPF_JGT: + return BPF_JLE; + case BPF_JGE: + return BPF_JLT; + case BPF_JLT: + return BPF_JGE; + case BPF_JLE: + return BPF_JGT; + case BPF_JSGT: + return BPF_JSLE; + case BPF_JSGE: + return BPF_JSLT; + case BPF_JSLT: + return BPF_JSGE; + case BPF_JSLE: + return BPF_JSGT; + } + return -1; +} + +static inline void cond_jump_offs16(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + switch (cond) { + case BPF_JEQ: + /* PC += jmp_offset if rj == rd */ + emit_insn(ctx, beq, rj, rd, jmp_offset); + return; + case BPF_JNE: + case BPF_JSET: + /* PC += jmp_offset if rj != rd */ + emit_insn(ctx, bne, rj, rd, jmp_offset); + return; + case BPF_JGT: + /* PC += jmp_offset if rj > rd (unsigned) */ + emit_insn(ctx, bltu, rd, rj, jmp_offset); + return; + case BPF_JLT: + /* PC += jmp_offset if rj < rd (unsigned) */ + emit_insn(ctx, bltu, rj, rd, jmp_offset); + return; + case BPF_JGE: + /* PC += jmp_offset if rj >= rd (unsigned) */ + emit_insn(ctx, bgeu, rj, rd, jmp_offset); + return; + case BPF_JLE: + /* PC += jmp_offset if rj <= rd (unsigned) */ + emit_insn(ctx, bgeu, rd, rj, jmp_offset); + return; + case BPF_JSGT: + /* PC += jmp_offset if rj > rd (signed) */ + emit_insn(ctx, blt, rd, rj, jmp_offset); + return; + case BPF_JSLT: + /* PC += jmp_offset if rj < rd (signed) */ + emit_insn(ctx, blt, rj, rd, jmp_offset); + return; + case BPF_JSGE: + /* PC += jmp_offset if rj >= rd (signed) */ + emit_insn(ctx, bge, rj, rd, jmp_offset); + return; + case BPF_JSLE: + /* PC += jmp_offset if rj <= rd (signed) */ + emit_insn(ctx, bge, rd, rj, jmp_offset); + return; + } +} + +static inline void cond_jump_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + cond = invert_jump_cond(cond); + cond_jump_offs16(ctx, cond, rj, rd, 2); + emit_insn(ctx, b, jmp_offset); +} + +static inline void cond_jump_offs32(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + s64 upper, lower; + + upper = (jmp_offset + (1 << 15)) >> 16; + lower = jmp_offset & 0xffff; + + cond = invert_jump_cond(cond); + cond_jump_offs16(ctx, cond, rj, rd, 3); + + /* + * jmp_addr = jmp_offset << 2 + * tmp2 = PC + jmp_addr[31, 18] + 18'b0 + */ + emit_insn(ctx, pcaddu18i, LOONGARCH_GPR_T2, upper << 2); + + /* jump to (tmp2 + jmp_addr[17, 2] + 2'b0) */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T2, lower + 1); +} + +static inline void uncond_jump_offs26(struct jit_ctx *ctx, int jmp_offset) +{ + emit_insn(ctx, b, jmp_offset); +} + +static inline void uncond_jump_offs32(struct jit_ctx *ctx, int jmp_offset, bool is_exit) +{ + s64 upper, lower; + + upper = (jmp_offset + (1 << 15)) >> 16; + lower = jmp_offset & 0xffff; + + if (is_exit) + lower -= 1; + + /* + * jmp_addr = jmp_offset << 2; + * tmp1 = PC + jmp_addr[31, 18] + 18'b0 + */ + emit_insn(ctx, pcaddu18i, LOONGARCH_GPR_T1, upper << 2); + + /* jump to (tmp1 + jmp_addr[17, 2] + 2'b0) */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, lower + 1); +} + +static inline void emit_cond_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + if (is_signed_imm16(jmp_offset)) + cond_jump_offs16(ctx, cond, rj, rd, jmp_offset); + else if (is_signed_imm26(jmp_offset)) + cond_jump_offs26(ctx, cond, rj, rd, jmp_offset); + else + cond_jump_offs32(ctx, cond, rj, rd, jmp_offset); +} + +static inline void emit_uncond_jump(struct jit_ctx *ctx, int jmp_offset, bool is_exit) +{ + if (is_signed_imm26(jmp_offset)) + uncond_jump_offs26(ctx, jmp_offset); + else + uncond_jump_offs32(ctx, jmp_offset, is_exit); +} + +static inline void emit_tailcall_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + if (is_signed_imm16(jmp_offset)) + cond_jump_offs16(ctx, cond, rj, rd, jmp_offset); + else if (is_signed_imm26(jmp_offset)) + cond_jump_offs26(ctx, cond, rj, rd, jmp_offset - 1); + else + cond_jump_offs32(ctx, cond, rj, rd, jmp_offset - 2); +} diff --git a/arch/loongarch/pci/Makefile b/arch/loongarch/pci/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8101ef3df71cf12f82a9bfdf356dd8e9f4d80443 --- /dev/null +++ b/arch/loongarch/pci/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the PCI specific kernel interface routines under Linux. +# + +obj-y += pci.o +obj-$(CONFIG_ACPI) += acpi.o diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c new file mode 100644 index 0000000000000000000000000000000000000000..9f418e40ecd003eb409ac762e20c28d3c9eeb1e3 --- /dev/null +++ b/arch/loongarch/pci/acpi.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct pci_root_info { + struct acpi_pci_root_info common; + struct pci_config_window *cfg; +}; + +void pcibios_add_bus(struct pci_bus *bus) +{ + acpi_pci_add_bus(bus); +} + +int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) +{ + struct pci_config_window *cfg = bridge->bus->sysdata; + struct acpi_device *adev = to_acpi_device(cfg->parent); + struct device *bus_dev = &bridge->bus->dev; + + ACPI_COMPANION_SET(&bridge->dev, adev); + set_dev_node(bus_dev, pa_to_nid(cfg->res.start)); + + return 0; +} + +int acpi_pci_bus_find_domain_nr(struct pci_bus *bus) +{ + struct pci_config_window *cfg = bus->sysdata; + struct acpi_device *adev = to_acpi_device(cfg->parent); + struct acpi_pci_root *root = acpi_driver_data(adev); + + return root->segment; +} + +static void acpi_release_root_info(struct acpi_pci_root_info *ci) +{ + struct pci_root_info *info; + + info = container_of(ci, struct pci_root_info, common); + pci_ecam_free(info->cfg); + kfree(ci->ops); + kfree(info); +} + +static int acpi_prepare_root_resources(struct acpi_pci_root_info *ci) +{ + struct acpi_device *device = ci->bridge; + struct resource_entry *entry, *tmp; + int status; + + status = acpi_pci_probe_root_resources(ci); + if (status > 0) { + resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { + if (entry->res->flags & IORESOURCE_MEM) { + entry->offset = ci->root->mcfg_addr & GENMASK_ULL(63, 40); + entry->res->start |= entry->offset; + entry->res->end |= entry->offset; + } + } + return status; + } + + resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { + dev_printk(KERN_DEBUG, &device->dev, + "host bridge window %pR (ignored)\n", entry->res); + resource_list_destroy_entry(entry); + } + + return 0; +} + +/* + * Create a PCI config space window + * - reserve mem region + * - alloc struct pci_config_window with space for all mappings + * - ioremap the config space + */ +static struct pci_config_window *arch_pci_ecam_create(struct device *dev, + struct resource *cfgres, struct resource *busr, const struct pci_ecam_ops *ops) +{ + int err; + unsigned int bus_range, bsz; + struct resource *conflict; + struct pci_config_window *cfg; + + if (busr->start > busr->end) + return ERR_PTR(-EINVAL); + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return ERR_PTR(-ENOMEM); + + cfg->parent = dev; + cfg->ops = ops; + cfg->busr.start = busr->start; + cfg->busr.end = busr->end; + cfg->busr.flags = IORESOURCE_BUS; + bus_range = resource_size(cfgres) >> ops->bus_shift; + + bsz = 1 << ops->bus_shift; + + cfg->res.start = cfgres->start; + cfg->res.end = cfgres->end; + cfg->res.flags = IORESOURCE_MEM | IORESOURCE_BUSY; + cfg->res.name = "PCI ECAM"; + + conflict = request_resource_conflict(&iomem_resource, &cfg->res); + if (conflict) { + err = -EBUSY; + dev_err(dev, "can't claim ECAM area %pR: address conflict with %s %pR\n", + &cfg->res, conflict->name, conflict); + goto err_exit; + } + + cfg->win = pci_remap_cfgspace(cfgres->start, bus_range * bsz); + if (!cfg->win) + goto err_exit_iomap; + + if (ops->init) { + err = ops->init(cfg); + if (err) + goto err_exit; + } + dev_info(dev, "ECAM at %pR for %pR\n", &cfg->res, &cfg->busr); + return cfg; + +err_exit_iomap: + err = -ENOMEM; + dev_err(dev, "ECAM ioremap failed\n"); +err_exit: + pci_ecam_free(cfg); + return ERR_PTR(err); +} + +/* + * Lookup the bus range for the domain in MCFG, and set up config space + * mapping. + */ +static struct pci_config_window * +pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) +{ + int ret, bus_shift; + u16 seg = root->segment; + struct device *dev = &root->device->dev; + struct resource cfgres; + struct resource *bus_res = &root->secondary; + struct pci_config_window *cfg; + const struct pci_ecam_ops *ecam_ops; + + ret = pci_mcfg_lookup(root, &cfgres, &ecam_ops); + if (ret < 0) { + dev_err(dev, "%04x:%pR ECAM region not found, use default value\n", seg, bus_res); + ecam_ops = &loongson_pci_ecam_ops; + root->mcfg_addr = mcfg_addr_init(0); + } + + bus_shift = ecam_ops->bus_shift ? : 20; + + if (bus_shift == 20) + cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops); + else { + cfgres.start = root->mcfg_addr + (bus_res->start << bus_shift); + cfgres.end = cfgres.start + (resource_size(bus_res) << bus_shift) - 1; + cfgres.end |= BIT(28) + (((PCI_CFG_SPACE_EXP_SIZE - 1) & 0xf00) << 16); + cfgres.flags = IORESOURCE_MEM; + cfg = arch_pci_ecam_create(dev, &cfgres, bus_res, ecam_ops); + } + + if (IS_ERR(cfg)) { + dev_err(dev, "%04x:%pR error %ld mapping ECAM\n", seg, bus_res, PTR_ERR(cfg)); + return NULL; + } + + return cfg; +} + +struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) +{ + struct pci_bus *bus; + struct pci_root_info *info; + struct acpi_pci_root_ops *root_ops; + int domain = root->segment; + int busnum = root->secondary.start; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + printk(KERN_WARNING "pci_bus %04x:%02x: " + "ignored (out of memory)\n", domain, busnum); + return NULL; + } + + root_ops = kzalloc(sizeof(*root_ops), GFP_KERNEL); + if (!root_ops) { + kfree(info); + return NULL; + } + + info->cfg = pci_acpi_setup_ecam_mapping(root); + if (!info->cfg) { + kfree(info); + kfree(root_ops); + return NULL; + } + + root_ops->release_info = acpi_release_root_info; + root_ops->prepare_resources = acpi_prepare_root_resources; + root_ops->pci_ops = (struct pci_ops *)&info->cfg->ops->pci_ops; + + bus = pci_find_bus(domain, busnum); + if (bus) { + memcpy(bus->sysdata, info->cfg, sizeof(struct pci_config_window)); + kfree(info); + } else { + struct pci_bus *child; + + bus = acpi_pci_root_create(root, root_ops, + &info->common, info->cfg); + if (!bus) { + kfree(info); + kfree(root_ops); + return NULL; + } + + pci_bus_size_bridges(bus); + pci_bus_assign_resources(bus); + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); + } + + return bus; +} diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c new file mode 100644 index 0000000000000000000000000000000000000000..2707c4d1e77240aa73abedcc5ea2e2553c296f24 --- /dev/null +++ b/arch/loongarch/pci/pci.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PCI_DEVICE_ID_LOONGSON_HOSTBRIDGE 0x7a00 + +int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn,int reg, int len, u32 *val) +{ + struct pci_bus * bus_tmp = pci_find_bus(domain, bus); + if (bus_tmp) + return bus_tmp->ops->read(bus_tmp, devfn, reg, len, val); + return -EINVAL; +} + +int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn, + int reg, int len, u32 val) +{ + struct pci_bus * bus_tmp = pci_find_bus(domain, bus); + if (bus_tmp) + return bus_tmp->ops->write(bus_tmp, devfn, reg, len, val); + return -EINVAL; +} + +phys_addr_t mcfg_addr_init(int node) +{ + return (((u64)node << 44) | MCFG_EXT_PCICFG_BASE); +} + +static int __init pcibios_init(void) +{ + unsigned int lsize; + + /* + * Set PCI cacheline size to that of the last level in the + * cache hierarchy. + */ + lsize = cpu_last_level_cache_line_size(); + + BUG_ON(!lsize); + + pci_dfl_cache_line_size = lsize >> 2; + + pr_debug("PCI: pci_cache_line_size set to %d bytes\n", lsize); + + return 0; +} + +subsys_initcall(pcibios_init); + +int pcibios_add_device(struct pci_dev *dev) +{ + int id = pci_domain_nr(dev->bus); + + dev_set_msi_domain(&dev->dev, pch_msi_domain[id]); + + return 0; +} + +int pcibios_alloc_irq(struct pci_dev *dev) +{ + if (acpi_disabled) + return 0; + if (pci_dev_msi_enabled(dev)) + return 0; + return acpi_pci_irq_enable(dev); +} + +static void pci_fixup_vgadev(struct pci_dev *pdev) +{ + struct pci_dev *devp = NULL; + + while ((devp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, devp))) { + if (devp->vendor != PCI_VENDOR_ID_LOONGSON) { + vga_set_default_device(devp); + dev_info(&pdev->dev, + "Overriding boot device as %X:%X\n", + devp->vendor, devp->device); + } + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, pci_fixup_vgadev); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, pci_fixup_vgadev); diff --git a/arch/loongarch/power/Makefile b/arch/loongarch/power/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..4eb720bac06d59ad7acc22b33609e8d0c220c328 --- /dev/null +++ b/arch/loongarch/power/Makefile @@ -0,0 +1,6 @@ +OBJECT_FILES_NON_STANDARD_suspend_asm.o := y + +obj-y += platform.o + +obj-$(CONFIG_SUSPEND) += suspend.o suspend_asm.o +obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate_asm.o diff --git a/arch/loongarch/power/hibernate.c b/arch/loongarch/power/hibernate.c new file mode 100644 index 0000000000000000000000000000000000000000..853b5d9c801b65d060337b3279b6004d8bbe43d8 --- /dev/null +++ b/arch/loongarch/power/hibernate.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include + +static u32 saved_crmd; +static u32 saved_prmd; +static u32 saved_euen; +static u32 saved_ecfg; +static u64 saved_pcpu_base; +struct pt_regs saved_regs; + +void save_processor_state(void) +{ + saved_crmd = csr_read32(LOONGARCH_CSR_CRMD); + saved_prmd = csr_read32(LOONGARCH_CSR_PRMD); + saved_euen = csr_read32(LOONGARCH_CSR_EUEN); + saved_ecfg = csr_read32(LOONGARCH_CSR_ECFG); + saved_pcpu_base = csr_read64(PERCPU_BASE_KS); + + if (is_fpu_owner()) + save_fp(current); +} + +void restore_processor_state(void) +{ + csr_write32(saved_crmd, LOONGARCH_CSR_CRMD); + csr_write32(saved_prmd, LOONGARCH_CSR_PRMD); + csr_write32(saved_euen, LOONGARCH_CSR_EUEN); + csr_write32(saved_ecfg, LOONGARCH_CSR_ECFG); + csr_write64(saved_pcpu_base, PERCPU_BASE_KS); + + if (is_fpu_owner()) + restore_fp(current); +} + +int pfn_is_nosave(unsigned long pfn) +{ + unsigned long nosave_begin_pfn = PFN_DOWN(__pa(&__nosave_begin)); + unsigned long nosave_end_pfn = PFN_UP(__pa(&__nosave_end)); + + return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); +} + +extern int swsusp_asm_suspend(void); + +int swsusp_arch_suspend(void) +{ + enable_pci_wakeup(); + return swsusp_asm_suspend(); +} + +extern int swsusp_asm_resume(void); + +int swsusp_arch_resume(void) +{ + /* Avoid TLB mismatch during and after kernel resume */ + local_flush_tlb_all(); + return swsusp_asm_resume(); +} diff --git a/arch/loongarch/power/hibernate_asm.S b/arch/loongarch/power/hibernate_asm.S new file mode 100644 index 0000000000000000000000000000000000000000..c606cae7756bd5a879d80456ccf954acc6cda74c --- /dev/null +++ b/arch/loongarch/power/hibernate_asm.S @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Hibernation support specific for loongarch - temporary page tables + * + * Licensed under the GPLv2 + * + * Copyright (C) 2009 Lemote Inc. + * Author: Hu Hongbing + * Wu Zhangjin + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include + +.text +SYM_FUNC_START(swsusp_asm_suspend) + la.pcrel t0, saved_regs + PTR_S ra, t0, PT_R1 + PTR_S tp, t0, PT_R2 + PTR_S sp, t0, PT_R3 + PTR_S u0, t0, PT_R21 + PTR_S fp, t0, PT_R22 + PTR_S s0, t0, PT_R23 + PTR_S s1, t0, PT_R24 + PTR_S s2, t0, PT_R25 + PTR_S s3, t0, PT_R26 + PTR_S s4, t0, PT_R27 + PTR_S s5, t0, PT_R28 + PTR_S s6, t0, PT_R29 + PTR_S s7, t0, PT_R30 + PTR_S s8, t0, PT_R31 + b swsusp_save +SYM_FUNC_END(swsusp_asm_suspend) + +SYM_FUNC_START(swsusp_asm_resume) + la.pcrel t0, restore_pblist + PTR_L t0, t0, 0 +0: + PTR_L t1, t0, PBE_ADDRESS /* source */ + PTR_L t2, t0, PBE_ORIG_ADDRESS /* destination */ + PTR_LI t3, _PAGE_SIZE + PTR_ADD t3, t3, t1 +1: + REG_L t8, t1, 0 + REG_S t8, t2, 0 + PTR_ADDI t1, t1, SZREG + PTR_ADDI t2, t2, SZREG + bne t1, t3, 1b + PTR_L t0, t0, PBE_NEXT + bnez t0, 0b + la.pcrel t0, saved_regs + PTR_L ra, t0, PT_R1 + PTR_L tp, t0, PT_R2 + PTR_L sp, t0, PT_R3 + PTR_L u0, t0, PT_R21 + PTR_L fp, t0, PT_R22 + PTR_L s0, t0, PT_R23 + PTR_L s1, t0, PT_R24 + PTR_L s2, t0, PT_R25 + PTR_L s3, t0, PT_R26 + PTR_L s4, t0, PT_R27 + PTR_L s5, t0, PT_R28 + PTR_L s6, t0, PT_R29 + PTR_L s7, t0, PT_R30 + PTR_L s8, t0, PT_R31 + PTR_LI a0, 0x0 + jirl zero, ra, 0 +SYM_FUNC_END(swsusp_asm_resume) diff --git a/arch/loongarch/power/platform.c b/arch/loongarch/power/platform.c new file mode 100644 index 0000000000000000000000000000000000000000..d923385dddb2131fae4ee2d69d534e605222174d --- /dev/null +++ b/arch/loongarch/power/platform.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Huacai Chen + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include + +#include +#include + +void enable_gpe_wakeup(void) +{ + acpi_enable_all_wakeup_gpes(); +} + +void enable_pci_wakeup(void) +{ + acpi_write_bit_register(ACPI_BITREG_PCIEXP_WAKE_STATUS, 1); + + if (acpi_gbl_FADT.flags & ACPI_FADT_PCI_EXPRESS_WAKE) + acpi_write_bit_register(ACPI_BITREG_PCIEXP_WAKE_DISABLE, 0); +} + +static struct platform_device loongson3_cpufreq_device = { + .name = "loongson3_cpufreq", + .id = -1, +}; + +static int __init loongson_cpufreq_init(void) +{ + return platform_device_register(&loongson3_cpufreq_device); +} + +arch_initcall(loongson_cpufreq_init); + +static int __init loongson3_acpi_suspend_init(void) +{ +#ifdef CONFIG_ACPI + acpi_status status; + uint64_t suspend_addr = 0; + + if (acpi_disabled || acpi_gbl_reduced_hardware) + return 0; + + acpi_write_bit_register(ACPI_BITREG_SCI_ENABLE, 1); + status = acpi_evaluate_integer(NULL, "\\SADR", NULL, &suspend_addr); + if (ACPI_FAILURE(status) || !suspend_addr) { + pr_err("ACPI S3 is not support!\n"); + return -1; + } + loongson_sysconf.suspend_addr = (u64)phys_to_virt(PHYSADDR(suspend_addr)); +#endif + return 0; +} + +device_initcall(loongson3_acpi_suspend_init); diff --git a/arch/loongarch/power/suspend.c b/arch/loongarch/power/suspend.c new file mode 100644 index 0000000000000000000000000000000000000000..e5b54bcfe043306a2ab3e8dbce1d30e609687b86 --- /dev/null +++ b/arch/loongarch/power/suspend.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * loongson-specific suspend support + * + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include + +#include +#include +#include +#include + +#include + +u64 loongarch_suspend_addr; + +struct saved_registers { + u32 ecfg; + u32 euen; + u64 pgd; + u64 kpgd; + u32 pwctl0; + u32 pwctl1; +}; +static struct saved_registers saved_regs; + +static void arch_common_suspend(void) +{ + save_counter(); + saved_regs.pgd = csr_read64(LOONGARCH_CSR_PGDL); + saved_regs.kpgd = csr_read64(LOONGARCH_CSR_PGDH); + saved_regs.pwctl0 = csr_read32(LOONGARCH_CSR_PWCTL0); + saved_regs.pwctl1 = csr_read32(LOONGARCH_CSR_PWCTL1); + saved_regs.ecfg = csr_read32(LOONGARCH_CSR_ECFG); + saved_regs.euen = csr_read32(LOONGARCH_CSR_EUEN); + + loongarch_suspend_addr = loongson_sysconf.suspend_addr; +} + +static void arch_common_resume(void) +{ + sync_counter(); + local_flush_tlb_all(); + csr_write64(per_cpu_offset(0), PERCPU_BASE_KS); + csr_write64(eentry, LOONGARCH_CSR_EENTRY); + csr_write64(eentry, LOONGARCH_CSR_MERRENTRY); + csr_write64(tlbrentry, LOONGARCH_CSR_TLBRENTRY); + + csr_write64(saved_regs.pgd, LOONGARCH_CSR_PGDL); + csr_write64(saved_regs.kpgd, LOONGARCH_CSR_PGDH); + csr_write32(saved_regs.pwctl0, LOONGARCH_CSR_PWCTL0); + csr_write32(saved_regs.pwctl1, LOONGARCH_CSR_PWCTL1); + csr_write32(saved_regs.ecfg, LOONGARCH_CSR_ECFG); + csr_write32(saved_regs.euen, LOONGARCH_CSR_EUEN); +} + +int loongarch_acpi_suspend(void) +{ + enable_gpe_wakeup(); + enable_pci_wakeup(); + + arch_common_suspend(); + + /* processor specific suspend */ + loongarch_suspend_enter(); + + arch_common_resume(); + + return 0; +} diff --git a/arch/loongarch/power/suspend_asm.S b/arch/loongarch/power/suspend_asm.S new file mode 100644 index 0000000000000000000000000000000000000000..49de8527f9bb9f82872fdfae42491c23832fc863 --- /dev/null +++ b/arch/loongarch/power/suspend_asm.S @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Sleep helper for Loongson-3 sleep mode. + * + * Copyright (C) 2020 Loongson Technology Co., Ltd. + * Author: Huacai Chen + */ + +#include +#include +#include +#include +#include + +/* preparatory stuff */ +.macro SETUP_SLEEP + addi.d sp, sp, -PT_SIZE + st.d $r1, sp, PT_R1 + st.d $r2, sp, PT_R2 + st.d $r3, sp, PT_R3 + st.d $r4, sp, PT_R4 + st.d $r21, sp, PT_R21 + st.d $r22, sp, PT_R22 + st.d $r23, sp, PT_R23 + st.d $r24, sp, PT_R24 + st.d $r25, sp, PT_R25 + st.d $r26, sp, PT_R26 + st.d $r27, sp, PT_R27 + st.d $r28, sp, PT_R28 + st.d $r29, sp, PT_R29 + st.d $r30, sp, PT_R30 + st.d $r31, sp, PT_R31 + + la.pcrel t0, acpi_saved_sp + st.d sp, t0, 0 +.endm + +.macro SETUP_WAKEUP + ld.d $r1, sp, PT_R1 + ld.d $r2, sp, PT_R2 + ld.d $r3, sp, PT_R3 + ld.d $r4, sp, PT_R4 + ld.d $r21, sp, PT_R21 + ld.d $r22, sp, PT_R22 + ld.d $r23, sp, PT_R23 + ld.d $r24, sp, PT_R24 + ld.d $r25, sp, PT_R25 + ld.d $r26, sp, PT_R26 + ld.d $r27, sp, PT_R27 + ld.d $r28, sp, PT_R28 + ld.d $r29, sp, PT_R29 + ld.d $r30, sp, PT_R30 + ld.d $r31, sp, PT_R31 +.endm + + .text + .align 12 + +/* Sleep/wakeup code for Loongson-3 */ +SYM_FUNC_START(loongarch_suspend_enter) + SETUP_SLEEP + bl __flush_cache_all + + /* Pass RA and SP to BIOS */ + addi.d a1, sp, 0 + la.pcrel a0, loongarch_wakeup_start + la.pcrel t0, loongarch_suspend_addr + ld.d t0, t0, 0 + jirl a0, t0, 0 /* Call BIOS's STR sleep routine */ + + /* + * This is where we return upon wakeup. + * Reload all of the registers and return. + */ +SYM_INNER_LABEL(loongarch_wakeup_start, SYM_L_GLOBAL) + li.d t0, CSR_DMW0_INIT # UC, PLV0 + csrwr t0, LOONGARCH_CSR_DMWIN0 + li.d t0, CSR_DMW1_INIT # CA, PLV0 + csrwr t0, LOONGARCH_CSR_DMWIN1 + + JUMP_VIRT_ADDR t0, t1 + + /* Enable PG */ + li.w t0, 0xb0 # PLV=0, IE=0, PG=1 + csrwr t0, LOONGARCH_CSR_CRMD + + la.pcrel t0, acpi_saved_sp + ld.d sp, t0, 0 + SETUP_WAKEUP + addi.d sp, sp, PT_SIZE + jr ra +SYM_FUNC_END(loongarch_suspend_enter) diff --git a/arch/loongarch/tools/Makefile b/arch/loongarch/tools/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0fc5c25937bafb36fefe05ae2cab0116838503dd --- /dev/null +++ b/arch/loongarch/tools/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +hostprogs := elf-entry +PHONY += elf-entry +elf-entry: $(obj)/elf-entry + @: diff --git a/arch/loongarch/tools/elf-entry.c b/arch/loongarch/tools/elf-entry.c new file mode 100644 index 0000000000000000000000000000000000000000..c80721e0dee186af3d1275679647feba54f92756 --- /dev/null +++ b/arch/loongarch/tools/elf-entry.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +__attribute__((noreturn)) +static void die(const char *msg) +{ + fputs(msg, stderr); + exit(EXIT_FAILURE); +} + +int main(int argc, const char *argv[]) +{ + uint64_t entry; + size_t nread; + FILE *file; + union { + Elf32_Ehdr ehdr32; + Elf64_Ehdr ehdr64; + } hdr; + + if (argc != 2) + die("Usage: elf-entry \n"); + + file = fopen(argv[1], "r"); + if (!file) { + perror("Unable to open input file"); + return EXIT_FAILURE; + } + + nread = fread(&hdr, 1, sizeof(hdr), file); + if (nread != sizeof(hdr)) { + fclose(file); + perror("Unable to read input file"); + return EXIT_FAILURE; + } + + if (memcmp(hdr.ehdr32.e_ident, ELFMAG, SELFMAG)) { + fclose(file); + die("Input is not an ELF\n"); + } + + switch (hdr.ehdr32.e_ident[EI_CLASS]) { + case ELFCLASS32: + /* Sign extend to form a canonical address */ + entry = (int64_t)(int32_t)hdr.ehdr32.e_entry; + break; + + case ELFCLASS64: + entry = hdr.ehdr64.e_entry; + break; + + default: + fclose(file); + die("Invalid ELF class\n"); + } + + fclose(file); + printf("0x%016" PRIx64 "\n", entry); + + return EXIT_SUCCESS; +} diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..6376f09fb2f3218b9d16048ab35bded478126c11 --- /dev/null +++ b/arch/loongarch/vdso/Makefile @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: GPL-2.0 +# Objects to go into the VDSO. +OBJECT_FILES_NON_STANDARD := y + +KASAN_SANITIZE := n + +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_LARCH_32|R_LARCH_64|R_LARCH_MARK_LA|R_LARCH_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile + +obj-vdso-y := elf.o vgetcpu.o vgettimeofday.o sigreturn.o + +# Common compiler flags between ABIs. +ccflags-vdso := \ + $(filter -I%,$(KBUILD_CFLAGS)) \ + $(filter -E%,$(KBUILD_CFLAGS)) \ + $(filter -march=%,$(KBUILD_CFLAGS)) \ + $(filter -m%-float,$(KBUILD_CFLAGS)) \ + -D__VDSO__ + +ifeq ($(cc-name),clang) +ccflags-vdso += $(filter --target=%,$(KBUILD_CFLAGS)) +endif + +cflags-vdso := $(ccflags-vdso) \ + $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ + -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ + -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ + $(call cc-option, -fno-asynchronous-unwind-tables) \ + $(call cc-option, -fno-stack-protector) +aflags-vdso := $(ccflags-vdso) \ + -D__ASSEMBLY__ -Wa,-gdwarf-2 + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + +# VDSO linker flags. +ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ + $(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared \ + --hash-style=sysv --build-id -T + +GCOV_PROFILE := n + +# +# Shared build commands. +# + +quiet_cmd_vdsold_and_vdso_check = LD $@ + cmd_vdsold_and_vdso_check = $(cmd_ld); $(cmd_vdso_check) + +quiet_cmd_vdsoas_o_S = AS $@ + cmd_vdsoas_o_S = $(CC) $(a_flags) -c -o $@ $< + +# Generate VDSO offsets using helper script +gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +quiet_cmd_vdsosym = VDSOSYM $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + +include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE + $(call if_changed,vdsosym) + +# +# Build native VDSO. +# + +native-abi := $(filter -mabi=%,$(KBUILD_CFLAGS)) + +targets += $(obj-vdso-y) +targets += vdso.lds vdso.so.dbg vdso.so + +obj-vdso := $(obj-vdso-y:%.o=$(obj)/%.o) + +$(obj-vdso): KBUILD_CFLAGS := $(cflags-vdso) $(native-abi) +$(obj-vdso): KBUILD_AFLAGS := $(aflags-vdso) $(native-abi) + +$(obj)/vdso.lds: KBUILD_CPPFLAGS := $(ccflags-vdso) $(native-abi) + +$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold_and_vdso_check) + +$(obj)/vdso.so: OBJCOPYFLAGS := -S +$(obj)/vdso.so: $(obj)/vdso.so.dbg FORCE + $(call if_changed,objcopy) + +obj-y += vdso.o + +$(obj)/vdso.o : $(obj)/vdso.so + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + +vdso_install: vdso.so diff --git a/arch/loongarch/vdso/elf.S b/arch/loongarch/vdso/elf.S new file mode 100644 index 0000000000000000000000000000000000000000..d5f16a6e3c726b2dc51f184525605ba6f7ea355e --- /dev/null +++ b/arch/loongarch/vdso/elf.S @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Huacai Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include + +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/loongarch/vdso/gen_vdso_offsets.sh b/arch/loongarch/vdso/gen_vdso_offsets.sh new file mode 100755 index 0000000000000000000000000000000000000000..1bb4e12642ff384d03713615c1101238c6a5a90e --- /dev/null +++ b/arch/loongarch/vdso/gen_vdso_offsets.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# +# Derived from RISC-V and ARM64: +# Author: Will Deacon +# +# Match symbols in the DSO that look like VDSO_*; produce a header file +# of constant offsets into the shared object. +# + +LC_ALL=C sed -n -e 's/^00*/0/' -e \ +'s/^\([0-9a-fA-F]*\) . VDSO_\([a-zA-Z0-9_]*\)$/\#define vdso_offset_\2\t0x\1/p' diff --git a/arch/loongarch/vdso/sigreturn.S b/arch/loongarch/vdso/sigreturn.S new file mode 100644 index 0000000000000000000000000000000000000000..98f765b20ba92cca4888a6206d023fe66e73fb57 --- /dev/null +++ b/arch/loongarch/vdso/sigreturn.S @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015 Imagination Technologies + * Author: Alex Smith + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include + +#include +#include + +#include +#include + + .section .text + .cfi_sections .debug_frame + +SYM_FUNC_START(__vdso_rt_sigreturn) + + li.w a7, __NR_rt_sigreturn + syscall 0 + +SYM_FUNC_END(__vdso_rt_sigreturn) diff --git a/arch/loongarch/vdso/vdso.S b/arch/loongarch/vdso/vdso.S new file mode 100644 index 0000000000000000000000000000000000000000..41951232ef4536d97d66ecbd245084e184bd4b16 --- /dev/null +++ b/arch/loongarch/vdso/vdso.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + * + * Derived from RISC-V: + * Copyright (C) 2014 Regents of the University of California + */ + +#include +#include +#include + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/loongarch/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/loongarch/vdso/vdso.lds.S b/arch/loongarch/vdso/vdso.lds.S new file mode 100644 index 0000000000000000000000000000000000000000..d437b9695ba5bee45827075d64f6f15e65563aa8 --- /dev/null +++ b/arch/loongarch/vdso/vdso.lds.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: Huacai Chen + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +OUTPUT_FORMAT("elf64-loongarch", "elf64-loongarch", "elf64-loongarch") + +OUTPUT_ARCH(loongarch) + +SECTIONS +{ + PROVIDE(_start = .); + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + .text : { *(.text*) } :text + PROVIDE (__etext = .); + PROVIDE (_etext = .); + PROVIDE (etext = .); + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .dynamic : { *(.dynamic) } :text :dynamic + + .rodata : { *(.rodata*) } :text + + _end = .; + PROVIDE(end = .); + + /DISCARD/ : { + *(.gnu.attributes) + *(.note.GNU-stack) + *(.data .data.* .gnu.linkonce.d.* .sdata*) + *(.bss .sbss .dynbss .dynsbss) + } +} + +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +VERSION +{ + LINUX_4.8 { + global: + __vdso_getcpu; + __vdso_clock_getres; + __vdso_clock_gettime; + __vdso_gettimeofday; + __vdso_rt_sigreturn; + local: *; + }; +} + +/* + * Make the sigreturn code visible to the kernel. + */ +VDSO_sigreturn = __vdso_rt_sigreturn; diff --git a/arch/loongarch/vdso/vgetcpu.c b/arch/loongarch/vdso/vgetcpu.c new file mode 100644 index 0000000000000000000000000000000000000000..5f9313ab3cb4c78b67bb7a8bedbb52807d916580 --- /dev/null +++ b/arch/loongarch/vdso/vgetcpu.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Fast user context implementation of getcpu() + */ + +#include +#include + +static __always_inline int read_cpu_id(void) +{ + int cpu_id; + + __asm__ __volatile__( + " rdtime.d $zero, %0\n" + : "=r" (cpu_id) + : + : "memory"); + + return cpu_id; +} + +static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void) +{ + return (struct vdso_pcpu_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START * PAGE_SIZE); +} + +int __vdso_getcpu(unsigned int *cpu, unsigned int *node, struct getcpu_cache *unused) +{ + int cpu_id; + const struct vdso_pcpu_data *data; + + cpu_id = read_cpu_id(); + + if (cpu) + *cpu = cpu_id; + + if (node) { + data = get_pcpu_data(); + *node = data[cpu_id].node; + } + + return 0; +} diff --git a/arch/loongarch/vdso/vgettimeofday.c b/arch/loongarch/vdso/vgettimeofday.c new file mode 100644 index 0000000000000000000000000000000000000000..c54489a630cf9662b7f2ed10c84dc761b1dc0897 --- /dev/null +++ b/arch/loongarch/vdso/vgettimeofday.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * LoongArch userspace implementations of gettimeofday() and similar. + * + * Copyright (C) 2020 Loongson Technologies + * + */ +#include +#include + +int __vdso_clock_gettime(clockid_t clock, + struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, + struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, + struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} diff --git a/arch/mips/alchemy/devboards/db1200.c b/arch/mips/alchemy/devboards/db1200.c index 414f92eacb5e5c6a94aea7a247ac7e7757c62c5b..9ad26215b00427585598817b73104a0cb62e8d8a 100644 --- a/arch/mips/alchemy/devboards/db1200.c +++ b/arch/mips/alchemy/devboards/db1200.c @@ -847,7 +847,7 @@ int __init db1200_dev_setup(void) i2c_register_board_info(0, db1200_i2c_devs, ARRAY_SIZE(db1200_i2c_devs)); spi_register_board_info(db1200_spi_devs, - ARRAY_SIZE(db1200_i2c_devs)); + ARRAY_SIZE(db1200_spi_devs)); /* SWITCHES: S6.8 I2C/SPI selector (OFF=I2C ON=SPI) * S6.7 AC97/I2S selector (OFF=AC97 ON=I2S) diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index 752b93d91ac9aa493a78a3e67b677c0087f87303..06811a5db71d73574b5c0eb0e4ce3b4d7c927a9d 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -588,7 +588,7 @@ int __init db1550_dev_setup(void) i2c_register_board_info(0, db1550_i2c_devs, ARRAY_SIZE(db1550_i2c_devs)); spi_register_board_info(db1550_spi_devs, - ARRAY_SIZE(db1550_i2c_devs)); + ARRAY_SIZE(db1550_spi_devs)); c = clk_get(NULL, "psc0_intclk"); if (!IS_ERR(c)) { diff --git a/arch/mips/include/asm/dmi.h b/arch/mips/include/asm/dmi.h index 27415a288adf56743f370e5017762d06ce3f5e5a..dc397f630c6608c85ba5224686e3470e5d5dc82e 100644 --- a/arch/mips/include/asm/dmi.h +++ b/arch/mips/include/asm/dmi.h @@ -5,7 +5,7 @@ #include #include -#define dmi_early_remap(x, l) ioremap_cache(x, l) +#define dmi_early_remap(x, l) ioremap(x, l) #define dmi_early_unmap(x, l) iounmap(x) #define dmi_remap(x, l) ioremap_cache(x, l) #define dmi_unmap(x) iounmap(x) diff --git a/arch/mips/kernel/elf.c b/arch/mips/kernel/elf.c index 7b045d2a0b51b56fc5aa705943aceb029199c50b..bbc6f07d81243562dab176bf811812a857593fb2 100644 --- a/arch/mips/kernel/elf.c +++ b/arch/mips/kernel/elf.c @@ -11,6 +11,7 @@ #include #include +#include #ifdef CONFIG_MIPS_FP_SUPPORT @@ -309,6 +310,11 @@ void mips_set_personality_nan(struct arch_elf_state *state) struct cpuinfo_mips *c = &boot_cpu_data; struct task_struct *t = current; + /* Do this early so t->thread.fpu.fcr31 won't be clobbered in case + * we are preempted before the lose_fpu(0) in start_thread. + */ + lose_fpu(0); + t->thread.fpu.fcr31 = c->fpu_csr31; switch (state->nan_2008) { case 0: diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index b7eb7dd96e179d48fd70e1fed7c93ed4ebb9533b..66643cb659630f4444347c3ef43f7e9126e99bdc 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -322,11 +322,11 @@ static void __init bootmem_init(void) panic("Incorrect memory mapping !!!"); if (max_pfn > PFN_DOWN(HIGHMEM_START)) { + max_low_pfn = PFN_DOWN(HIGHMEM_START); #ifdef CONFIG_HIGHMEM - highstart_pfn = PFN_DOWN(HIGHMEM_START); + highstart_pfn = max_low_pfn; highend_pfn = max_pfn; #else - max_low_pfn = PFN_DOWN(HIGHMEM_START); max_pfn = max_low_pfn; #endif } diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index 7ed28ddcaba7d8a84c4f209bf4976008aba4e0e0..25050b0ab6fde26c3d2aeef91e3b95d0ef7c55b2 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -123,10 +123,10 @@ static unsigned long f_extend(unsigned long address) #ifdef CONFIG_64BIT if(unlikely(parisc_narrow_firmware)) { if((address & 0xff000000) == 0xf0000000) - return 0xf0f0f0f000000000UL | (u32)address; + return (0xfffffff0UL << 32) | (u32)address; if((address & 0xf0000000) == 0xf0000000) - return 0xffffffff00000000UL | (u32)address; + return (0xffffffffUL << 32) | (u32)address; } #endif return address; diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 255a1837e9f7fb50514c0e3e69fd5f16a47a87ad..3a5a27318a0e511f3a08fe1fdc7abc46dbcfdde9 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -390,5 +390,9 @@ extern void *abatron_pteptrs[2]; #include #endif +#if defined(CONFIG_FA_DUMP) || defined(CONFIG_PRESERVE_FA_DUMP) +#define __HAVE_ARCH_RESERVED_KERNEL_PAGES +#endif + #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MMU_H_ */ diff --git a/arch/powerpc/include/asm/mmzone.h b/arch/powerpc/include/asm/mmzone.h index 6cda76b57c5dd044a0db10783882a33927245927..bd1a8d7256ff2e1b24a80ae50840c607a8b0c41d 100644 --- a/arch/powerpc/include/asm/mmzone.h +++ b/arch/powerpc/include/asm/mmzone.h @@ -42,9 +42,6 @@ u64 memory_hotplug_max(void); #else #define memory_hotplug_max() memblock_end_of_DRAM() #endif /* CONFIG_NEED_MULTIPLE_NODES */ -#ifdef CONFIG_FA_DUMP -#define __HAVE_ARCH_RESERVED_KERNEL_PAGES -#endif #ifdef CONFIG_MEMORY_HOTPLUG extern int create_section_mapping(unsigned long start, unsigned long end, diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 5e5a2448ae79a6281f2a0662d107dc37c1ed090f..b0e87dce2b9a0cb0d0037da5ddeefc991f10a649 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -1432,10 +1432,12 @@ static int emulate_instruction(struct pt_regs *regs) return -EINVAL; } +#ifdef CONFIG_GENERIC_BUG int is_valid_bugaddr(unsigned long addr) { return is_kernel_addr(addr); } +#endif #ifdef CONFIG_MATH_EMULATION static int emulate_math(struct pt_regs *regs) diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index db17e5f5d431b341e4eba0da04f56e8ab2feb46b..3ac43637103ff78c115c9843b57ae6e363b330a0 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -250,7 +250,7 @@ int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) return -ENOMEM; - p->pfns = vzalloc(array_size(slot->npages, sizeof(*p->pfns))); + p->pfns = vcalloc(slot->npages, sizeof(*p->pfns)); if (!p->pfns) { kfree(p); return -ENOMEM; diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index 2d19655328f12c7d7b9c3141b7251b71874c3444..ca4733fbd02deee03825c2a4f8acba6aa998da2e 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -512,6 +512,8 @@ static int do_fp_load(struct instruction_op *op, unsigned long ea, } u; nb = GETSIZE(op->type); + if (nb > sizeof(u)) + return -EINVAL; if (!address_ok(regs, ea, nb)) return -EFAULT; rn = op->reg; @@ -562,6 +564,8 @@ static int do_fp_store(struct instruction_op *op, unsigned long ea, } u; nb = GETSIZE(op->type); + if (nb > sizeof(u)) + return -EINVAL; if (!address_ok(regs, ea, nb)) return -EFAULT; rn = op->reg; @@ -606,6 +610,9 @@ static nokprobe_inline int do_vec_load(int rn, unsigned long ea, u8 b[sizeof(__vector128)]; } u = {}; + if (size > sizeof(u)) + return -EINVAL; + if (!address_ok(regs, ea & ~0xfUL, 16)) return -EFAULT; /* align to multiple of size */ @@ -633,6 +640,9 @@ static nokprobe_inline int do_vec_store(int rn, unsigned long ea, u8 b[sizeof(__vector128)]; } u; + if (size > sizeof(u)) + return -EINVAL; + if (!address_ok(regs, ea & ~0xfUL, 16)) return -EFAULT; /* align to multiple of size */ diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index e18ae50a275c8fde6a64cc083bf44074f7ba7f0c..a86d932a7c3061ca1768952b64a0bad38c647367 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -446,6 +446,7 @@ void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, set_pte_at(vma->vm_mm, addr, ptep, pte); } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* * For hash translation mode, we use the deposited table to store hash slot * information and they are stored at PTRS_PER_PMD offset from related pmd @@ -467,6 +468,7 @@ int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl, return true; } +#endif /* * Does the CPU support tlbie? diff --git a/arch/powerpc/mm/init-common.c b/arch/powerpc/mm/init-common.c index 8e0d792ac2967c029f1c29a6254d953e92342771..52a20c97e46eda17c5e43530876f0ded9dc9282f 100644 --- a/arch/powerpc/mm/init-common.c +++ b/arch/powerpc/mm/init-common.c @@ -111,7 +111,7 @@ void pgtable_cache_add(unsigned int shift) * as to leave enough 0 bits in the address to contain it. */ unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1, HUGEPD_SHIFT_MASK + 1); - struct kmem_cache *new; + struct kmem_cache *new = NULL; /* It would be nice if this was a BUILD_BUG_ON(), but at the * moment, gcc doesn't seem to recognize is_power_of_2 as a @@ -124,7 +124,8 @@ void pgtable_cache_add(unsigned int shift) align = max_t(unsigned long, align, minalign); name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); - new = kmem_cache_create(name, table_size, align, 0, ctor(shift)); + if (name) + new = kmem_cache_create(name, table_size, align, 0, ctor(shift)); if (!new) panic("Could not allocate pgtable cache for order %d", shift); diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 812730e6bfffd2b460f5d5e40fac046f318397a0..8ef4e5034765be2eba56a5cb6e7f6f8a7fa0f7ea 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -600,7 +600,9 @@ static int ctr_aes_crypt(struct skcipher_request *req) * final block may be < AES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { - cpacf_kmctr(sctx->fc, sctx->key, buf, walk.src.virt.addr, + memset(buf, 0, AES_BLOCK_SIZE); + memcpy(buf, walk.src.virt.addr, nbytes); + cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); memcpy(walk.dst.virt.addr, buf, nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index a6727ad58d65ac3cf37f674ee34594e2101c0795..6a0a22621671ce6bfc6bbcc16c19735a82e6642c 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -676,9 +676,11 @@ static int ctr_paes_crypt(struct skcipher_request *req) * final block may be < AES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { + memset(buf, 0, AES_BLOCK_SIZE); + memcpy(buf, walk.src.virt.addr, nbytes); while (1) { if (cpacf_kmctr(ctx->fc, ¶m, buf, - walk.src.virt.addr, AES_BLOCK_SIZE, + buf, AES_BLOCK_SIZE, walk.iv) == AES_BLOCK_SIZE) break; if (__paes_convert_key(ctx)) diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h index 287bb88f76986e127388efd03c18d117bf4c417e..2686bee800e3d5a35f2d4918aac38f4020e0889c 100644 --- a/arch/s390/include/asm/pci_io.h +++ b/arch/s390/include/asm/pci_io.h @@ -11,6 +11,8 @@ /* I/O size constraints */ #define ZPCI_MAX_READ_SIZE 8 #define ZPCI_MAX_WRITE_SIZE 128 +#define ZPCI_BOUNDARY_SIZE (1 << 12) +#define ZPCI_BOUNDARY_MASK (ZPCI_BOUNDARY_SIZE - 1) /* I/O Map */ #define ZPCI_IOMAP_SHIFT 48 @@ -125,16 +127,18 @@ static inline int zpci_read_single(void *dst, const volatile void __iomem *src, int zpci_write_block(volatile void __iomem *dst, const void *src, unsigned long len); -static inline u8 zpci_get_max_write_size(u64 src, u64 dst, int len, int max) +static inline int zpci_get_max_io_size(u64 src, u64 dst, int len, int max) { - int count = len > max ? max : len, size = 1; + int offset = dst & ZPCI_BOUNDARY_MASK; + int size; - while (!(src & 0x1) && !(dst & 0x1) && ((size << 1) <= count)) { - dst = dst >> 1; - src = src >> 1; - size = size << 1; - } - return size; + size = min3(len, ZPCI_BOUNDARY_SIZE - offset, max); + if (IS_ALIGNED(src, 8) && IS_ALIGNED(dst, 8) && IS_ALIGNED(size, 8)) + return size; + + if (size >= 8) + return 8; + return rounddown_pow_of_two(size); } static inline int zpci_memcpy_fromio(void *dst, @@ -144,9 +148,9 @@ static inline int zpci_memcpy_fromio(void *dst, int size, rc = 0; while (n > 0) { - size = zpci_get_max_write_size((u64 __force) src, - (u64) dst, n, - ZPCI_MAX_READ_SIZE); + size = zpci_get_max_io_size((u64 __force) src, + (u64) dst, n, + ZPCI_MAX_READ_SIZE); rc = zpci_read_single(dst, src, size); if (rc) break; @@ -166,9 +170,9 @@ static inline int zpci_memcpy_toio(volatile void __iomem *dst, return -EINVAL; while (n > 0) { - size = zpci_get_max_write_size((u64 __force) dst, - (u64) src, n, - ZPCI_MAX_WRITE_SIZE); + size = zpci_get_max_io_size((u64 __force) dst, + (u64) src, n, + ZPCI_MAX_WRITE_SIZE); if (size > 8) /* main path */ rc = zpci_write_block(dst, src, size); else diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 3009bb52725242a795471710abd32b6a5a80f8ef..f381caddd9059ae66735b7134d7c3a1ff660846d 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -411,6 +411,7 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) /* * floating point control reg. is in the thread structure */ + save_fpu_regs(); if ((unsigned int) data != 0 || test_fp_ctl(data >> (BITS_PER_LONG - 32))) return -EINVAL; @@ -771,6 +772,7 @@ static int __poke_user_compat(struct task_struct *child, /* * floating point control reg. is in the thread structure */ + save_fpu_regs(); if (test_fp_ctl(tmp)) return -EINVAL; child->thread.fpu.fpc = data; @@ -1010,9 +1012,7 @@ static int s390_fpregs_set(struct task_struct *target, int rc = 0; freg_t fprs[__NUM_FPRS]; - if (target == current) - save_fpu_regs(); - + save_fpu_regs(); if (MACHINE_HAS_VX) convert_vx_to_fp(fprs, target->thread.fpu.vxrs); else diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 7a326d03087abd0455774ce0e9be46ba6d1c4e69..f6c27b44766f066653aa369b4bd63debca109103 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3649,10 +3649,6 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) vcpu_load(vcpu); - if (test_fp_ctl(fpu->fpc)) { - ret = -EINVAL; - goto out; - } vcpu->run->s.regs.fpc = fpu->fpc; if (MACHINE_HAS_VX) convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs, @@ -3660,7 +3656,6 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) else memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs)); -out: vcpu_put(vcpu); return ret; } diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 1ec8076209cab37018dfcd7cd5972121376bcc97..6e7c4762bd231eab3dc094c6ccd3c1ac8b2d21d1 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -100,9 +100,9 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, old_fs = enable_sacf_uaccess(); while (n > 0) { - size = zpci_get_max_write_size((u64 __force) dst, - (u64 __force) src, n, - ZPCI_MAX_WRITE_SIZE); + size = zpci_get_max_io_size((u64 __force) dst, + (u64 __force) src, n, + ZPCI_MAX_WRITE_SIZE); if (size > 8) /* main path */ rc = __pcistb_mio_inuser(dst, src, size, &status); else @@ -252,9 +252,9 @@ static inline int __memcpy_fromio_inuser(void __user *dst, old_fs = enable_sacf_uaccess(); while (n > 0) { - size = zpci_get_max_write_size((u64 __force) src, - (u64 __force) dst, n, - ZPCI_MAX_READ_SIZE); + size = zpci_get_max_io_size((u64 __force) src, + (u64 __force) dst, n, + ZPCI_MAX_READ_SIZE); rc = __pcilg_mio_inuser(dst, src, size, &status); if (rc) break; diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index f4a2e6d373b29a283c95f242d7acdeb346022562..1e4eb3894ec4d62e6b05e4f2052f2c6e59b229a8 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -367,3 +367,8 @@ void do_boot_page_fault(struct pt_regs *regs, unsigned long error_code) */ add_identity_map(address, end); } + +void do_boot_nmi_trap(struct pt_regs *regs, unsigned long error_code) +{ + /* Empty handler to ignore NMI during early boot */ +} diff --git a/arch/x86/boot/compressed/idt_64.c b/arch/x86/boot/compressed/idt_64.c index 804a502ee0d28bae40d754a00d282a5aa8ffeb51..eb30bb20c33b3de5ee9abe2de3b17d5010e864d3 100644 --- a/arch/x86/boot/compressed/idt_64.c +++ b/arch/x86/boot/compressed/idt_64.c @@ -45,6 +45,7 @@ void load_stage2_idt(void) boot_idt_desc.address = (unsigned long)boot_idt; set_idt_entry(X86_TRAP_PF, boot_page_fault); + set_idt_entry(X86_TRAP_NMI, boot_nmi_trap); #ifdef CONFIG_AMD_MEM_ENCRYPT set_idt_entry(X86_TRAP_VC, boot_stage2_vc); diff --git a/arch/x86/boot/compressed/idt_handlers_64.S b/arch/x86/boot/compressed/idt_handlers_64.S index 22890e199f5b44c7bc8f617f266e4364917c8efc..4d03c8562f637d2eb473f23d24e7dae61d730cb2 100644 --- a/arch/x86/boot/compressed/idt_handlers_64.S +++ b/arch/x86/boot/compressed/idt_handlers_64.S @@ -70,6 +70,7 @@ SYM_FUNC_END(\name) .code64 EXCEPTION_HANDLER boot_page_fault do_boot_page_fault error_code=1 +EXCEPTION_HANDLER boot_nmi_trap do_boot_nmi_trap error_code=0 #ifdef CONFIG_AMD_MEM_ENCRYPT EXCEPTION_HANDLER boot_stage1_vc do_vc_no_ghcb error_code=1 diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index d9a631c5973c7f6423299baae66167579a8bdcff..0ccc327184831f4407fc20ee54bbb0b6306c46d7 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -156,6 +156,7 @@ extern struct desc_ptr boot_idt_desc; /* IDT Entry Points */ void boot_page_fault(void); +void boot_nmi_trap(void); void boot_stage1_vc(void); void boot_stage2_vc(void); diff --git a/arch/x86/include/asm/syscall_wrapper.h b/arch/x86/include/asm/syscall_wrapper.h index a84333adeef232aa4d38f10d8ec1af7c1155ddb7..a507be368927500f9b1ff256e993555ebeb4bba5 100644 --- a/arch/x86/include/asm/syscall_wrapper.h +++ b/arch/x86/include/asm/syscall_wrapper.h @@ -58,12 +58,29 @@ extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); ,,regs->di,,regs->si,,regs->dx \ ,,regs->r10,,regs->r8,,regs->r9) \ + +/* SYSCALL_PT_ARGS is Adapted from s390x */ +#define SYSCALL_PT_ARG6(m, t1, t2, t3, t4, t5, t6) \ + SYSCALL_PT_ARG5(m, t1, t2, t3, t4, t5), m(t6, (regs->bp)) +#define SYSCALL_PT_ARG5(m, t1, t2, t3, t4, t5) \ + SYSCALL_PT_ARG4(m, t1, t2, t3, t4), m(t5, (regs->di)) +#define SYSCALL_PT_ARG4(m, t1, t2, t3, t4) \ + SYSCALL_PT_ARG3(m, t1, t2, t3), m(t4, (regs->si)) +#define SYSCALL_PT_ARG3(m, t1, t2, t3) \ + SYSCALL_PT_ARG2(m, t1, t2), m(t3, (regs->dx)) +#define SYSCALL_PT_ARG2(m, t1, t2) \ + SYSCALL_PT_ARG1(m, t1), m(t2, (regs->cx)) +#define SYSCALL_PT_ARG1(m, t1) m(t1, (regs->bx)) +#define SYSCALL_PT_ARGS(x, ...) SYSCALL_PT_ARG##x(__VA_ARGS__) + +#define __SC_COMPAT_CAST(t, a) \ + (__typeof(__builtin_choose_expr(__TYPE_IS_L(t), 0, 0U))) \ + (unsigned int)a + /* Mapping of registers to parameters for syscalls on i386 */ #define SC_IA32_REGS_TO_ARGS(x, ...) \ - __MAP(x,__SC_ARGS \ - ,,(unsigned int)regs->bx,,(unsigned int)regs->cx \ - ,,(unsigned int)regs->dx,,(unsigned int)regs->si \ - ,,(unsigned int)regs->di,,(unsigned int)regs->bp) + SYSCALL_PT_ARGS(x, __SC_COMPAT_CAST, \ + __MAP(x, __SC_TYPE, __VA_ARGS__)) \ #define __SYS_STUB0(abi, name) \ long __##abi##_##name(const struct pt_regs *regs); \ diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 0b7c81389c50a9457f458287f0f52b500486710b..18a6ed2afca035044da4861cdc71267cdbc7af9d 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -274,6 +275,7 @@ static noinstr void mce_panic(const char *msg, struct mce *final, char *exp) struct llist_node *pending; struct mce_evt_llist *l; int apei_err = 0; + struct page *p; /* * Allow instrumentation around external facilities usage. Not that it @@ -329,6 +331,20 @@ static noinstr void mce_panic(const char *msg, struct mce *final, char *exp) if (!fake_panic) { if (panic_timeout == 0) panic_timeout = mca_cfg.panic_timeout; + + /* + * Kdump skips the poisoned page in order to avoid + * touching the error bits again. Poison the page even + * if the error is fatal and the machine is about to + * panic. + */ + if (kexec_crash_loaded()) { + if (final && (final->status & MCI_STATUS_ADDRV)) { + p = pfn_to_online_page(final->addr >> PAGE_SHIFT); + if (p) + SetPageHWPoison(p); + } + } panic(msg); } else pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index bb657e2e6c6875c9c851d8176624972d4c2d6c84..b79b4ee49fbd0daa8d43cd7e82c2d339ce122860 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -24,8 +24,8 @@ static int kvmclock __initdata = 1; static int kvmclock_vsyscall __initdata = 1; -static int msr_kvm_system_time __ro_after_init = MSR_KVM_SYSTEM_TIME; -static int msr_kvm_wall_clock __ro_after_init = MSR_KVM_WALL_CLOCK; +static int msr_kvm_system_time __ro_after_init; +static int msr_kvm_wall_clock __ro_after_init; static u64 kvm_sched_clock_offset __ro_after_init; static int __init parse_no_kvmclock(char *arg) @@ -196,7 +196,8 @@ static void kvm_setup_secondary_clock(void) void kvmclock_disable(void) { - native_write_msr(msr_kvm_system_time, 0, 0); + if (msr_kvm_system_time) + native_write_msr(msr_kvm_system_time, 0, 0); } static void __init kvmclock_init_mem(void) @@ -292,7 +293,10 @@ void __init kvmclock_init(void) if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) { msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW; msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW; - } else if (!kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) { + } else if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) { + msr_kvm_system_time = MSR_KVM_SYSTEM_TIME; + msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK; + } else { return; } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 81cf4babbd0b475cddc204267b9ea87503b79e3f..3c379335ea4772b0ac728aeb3de1aa7806822e5f 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -35,7 +35,7 @@ int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { slot->arch.gfn_track[i] = - kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), + __vcalloc(npages, sizeof(*slot->arch.gfn_track[i]), GFP_KERNEL_ACCOUNT); if (!slot->arch.gfn_track[i]) goto track_free; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 890cb4c8a65bdf837b016042face4579252f5dab..8e0b957c62193d56ecc2abe71f336cb961ebc1c0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10829,14 +10829,14 @@ static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, slot->base_gfn, level) + 1; slot->arch.rmap[i] = - kvcalloc(lpages, sizeof(*slot->arch.rmap[i]), + __vcalloc(lpages, sizeof(*slot->arch.rmap[i]), GFP_KERNEL_ACCOUNT); if (!slot->arch.rmap[i]) goto out_free; if (i == 0) continue; - linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); + linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); if (!linfo) goto out_free; diff --git a/block/bio.c b/block/bio.c index 3caf222b50d5d620a92ccab4f7cbe6f31f62dac4..f834b14971564d98e5a8355ef86ea6a4c6c10a05 100644 --- a/block/bio.c +++ b/block/bio.c @@ -954,7 +954,7 @@ void bio_release_pages(struct bio *bio, bool mark_dirty) return; bio_for_each_segment_all(bvec, bio, iter_all) { - if (mark_dirty && !PageCompound(bvec->bv_page)) + if (mark_dirty) set_page_dirty_lock(bvec->bv_page); put_page(bvec->bv_page); } @@ -1326,8 +1326,7 @@ void bio_set_pages_dirty(struct bio *bio) struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { - if (!PageCompound(bvec->bv_page)) - set_page_dirty_lock(bvec->bv_page); + set_page_dirty_lock(bvec->bv_page); } } @@ -1375,7 +1374,7 @@ void bio_check_pages_dirty(struct bio *bio) struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { - if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) + if (!PageDirty(bvec->bv_page)) goto defer; } diff --git a/crypto/Kconfig b/crypto/Kconfig index 3564fe8d17ee531f3b9b36e836ab54f9389ffc7f..220581f3c062b00030d795096443544398b55c90 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -663,6 +663,14 @@ config CRYPTO_CRC32_MIPS CRC32c and CRC32 CRC algorithms implemented using mips crypto instructions, when available. +config CRYPTO_CRC32_LOONGARCH + tristate "CRC32c and CRC32 CRC algorithm (LoongArch)" + depends on LOONGARCH + select CRC32 + select CRYPTO_HASH + help + CRC32c and CRC32 CRC algorithms implemented using LoongArch + CRC32 instructions, when available. config CRYPTO_XXHASH tristate "xxHash hash algorithm" diff --git a/crypto/aead.c b/crypto/aead.c index 16991095270d2f6777a875d1417b07d92506b039..c4ece86c45bc4ed8c8d1495c716223936e62510a 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -35,8 +35,7 @@ static int setkey_unaligned(struct crypto_aead *tfm, const u8 *key, alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); ret = crypto_aead_alg(tfm)->setkey(tfm, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + kfree_sensitive(buffer); return ret; } diff --git a/crypto/algapi.c b/crypto/algapi.c index 5d422e725b267dadfa00fb3b5549ef798dbbe206..bb03bed14f740ccd121040c89af831eb0bd9c27f 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -258,6 +258,7 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) } if (!strcmp(q->cra_driver_name, alg->cra_name) || + !strcmp(q->cra_driver_name, alg->cra_driver_name) || !strcmp(q->cra_name, alg->cra_driver_name)) goto err; } diff --git a/crypto/cipher.c b/crypto/cipher.c index fd78150deb1c14c6830df06b4511df2e4802d293..72c5606cc7f813c6aa0cf97c449eb74141353435 100644 --- a/crypto/cipher.c +++ b/crypto/cipher.c @@ -33,8 +33,7 @@ static int setkey_unaligned(struct crypto_cipher *tfm, const u8 *key, alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); ret = cia->cia_setkey(crypto_cipher_tfm(tfm), alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + kfree_sensitive(buffer); return ret; } diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index b5ea34c340ccc77211bff18a97ca1cf432852e17..aff14992598ce87150eccbaf2f395148bef94525 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -272,9 +272,9 @@ config ACPI_CPPC_LIB config ACPI_PROCESSOR tristate "Processor" - depends on X86 || IA64 || ARM64 + depends on X86 || IA64 || ARM64 || LOONGARCH select ACPI_PROCESSOR_IDLE - select ACPI_CPU_FREQ_PSS if X86 || IA64 + select ACPI_CPU_FREQ_PSS if X86 || IA64 || LOONGARCH default y help This driver adds support for the ACPI Processor package. It is required @@ -514,10 +514,10 @@ config ACPI_CONFIGFS if ARM64 source "drivers/acpi/arm64/Kconfig" +endif config ACPI_PPTT bool -endif source "drivers/acpi/pmic/Kconfig" diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c index 088db2356998f52a0672f9fc4f8477b98134006d..0a84d5afd37c1f1cf51f837eccdd0d6e3679b22f 100644 --- a/drivers/acpi/acpi_extlog.c +++ b/drivers/acpi/acpi_extlog.c @@ -308,9 +308,10 @@ static int __init extlog_init(void) static void __exit extlog_exit(void) { mce_unregister_decode_chain(&extlog_mce_dec); - ((struct extlog_l1_head *)extlog_l1_addr)->flags &= ~FLAG_OS_OPTIN; - if (extlog_l1_addr) + if (extlog_l1_addr) { + ((struct extlog_l1_head *)extlog_l1_addr)->flags &= ~FLAG_OS_OPTIN; acpi_os_unmap_iomem(extlog_l1_addr, l1_size); + } if (elog_addr) acpi_os_unmap_iomem(elog_addr, elog_size); release_mem_region(elog_base, elog_size); diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 9d384656323a956f8458819b749c97f71d70eb6d..b2364ac455f34e2d4794c84363c84c629562ef46 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -568,6 +568,15 @@ static const struct dmi_system_id video_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Vostro 3350"), }, }, + { + .callback = video_set_report_key_events, + .driver_data = (void *)((uintptr_t)REPORT_BRIGHTNESS_KEY_EVENTS), + .ident = "COLORFUL X15 AT 23", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "COLORFUL"), + DMI_MATCH(DMI_PRODUCT_NAME, "X15 AT 23"), + }, + }, /* * Some machines change the brightness themselves when a brightness * hotkey gets pressed, despite us telling them not to. In this case diff --git a/drivers/acpi/acpica/evevent.c b/drivers/acpi/acpica/evevent.c index 9efca54c51ac50c6638f0f19c41a8f3efb8ef696..50c0e2f1e75822cbf6bb66bc7010bac6549124e3 100644 --- a/drivers/acpi/acpica/evevent.c +++ b/drivers/acpi/acpica/evevent.c @@ -140,9 +140,8 @@ static acpi_status acpi_ev_fixed_event_initialize(void) if (acpi_gbl_fixed_event_info[i].enable_register_id != 0xFF) { status = - acpi_write_bit_register(acpi_gbl_fixed_event_info - [i].enable_register_id, - ACPI_DISABLE_EVENT); + acpi_write_bit_register(acpi_gbl_fixed_event_info[i].enable_register_id, + (i == ACPI_EVENT_PCIE_WAKE) ? ACPI_ENABLE_EVENT : ACPI_DISABLE_EVENT); if (ACPI_FAILURE(status)) { return (status); } @@ -185,6 +184,11 @@ u32 acpi_ev_fixed_event_detect(void) return (int_status); } + if (fixed_enable & ACPI_BITMASK_PCIEXP_WAKE_DISABLE) + fixed_enable &= ~ACPI_BITMASK_PCIEXP_WAKE_DISABLE; + else + fixed_enable |= ACPI_BITMASK_PCIEXP_WAKE_DISABLE; + ACPI_DEBUG_PRINT((ACPI_DB_INTERRUPTS, "Fixed Event Block: Enable %08X Status %08X\n", fixed_enable, fixed_status)); @@ -250,7 +254,7 @@ static u32 acpi_ev_fixed_event_dispatch(u32 event) if (!acpi_gbl_fixed_event_handlers[event].handler) { (void)acpi_write_bit_register(acpi_gbl_fixed_event_info[event]. enable_register_id, - ACPI_DISABLE_EVENT); + event == ACPI_EVENT_PCIE_WAKE ? ACPI_ENABLE_EVENT : ACPI_DISABLE_EVENT); ACPI_ERROR((AE_INFO, "No installed handler for fixed event - %s (%u), disabling", diff --git a/drivers/acpi/acpica/hwsleep.c b/drivers/acpi/acpica/hwsleep.c index 6a20bb5059c1d815a33ab83149dca348065e5a44..e89895622a72fc9251a90937adfdcbbe5d82f8d8 100644 --- a/drivers/acpi/acpica/hwsleep.c +++ b/drivers/acpi/acpica/hwsleep.c @@ -311,6 +311,18 @@ acpi_status acpi_hw_legacy_wake(u8 sleep_state) [ACPI_EVENT_SLEEP_BUTTON]. status_register_id, ACPI_CLEAR_STATUS); + /* Enable pcie wake event if support */ + if ((acpi_gbl_FADT.flags & ACPI_FADT_PCI_EXPRESS_WAKE)) { + (void) + acpi_write_bit_register(acpi_gbl_fixed_event_info + [ACPI_EVENT_PCIE_WAKE]. + enable_register_id, ACPI_DISABLE_EVENT); + (void) + acpi_write_bit_register(acpi_gbl_fixed_event_info + [ACPI_EVENT_PCIE_WAKE]. + status_register_id, ACPI_CLEAR_STATUS); + } + acpi_hw_execute_sleep_method(METHOD_PATHNAME__SST, ACPI_SST_WORKING); return_ACPI_STATUS(status); } diff --git a/drivers/acpi/acpica/utglobal.c b/drivers/acpi/acpica/utglobal.c index e6dcbdc3fc6ec8e3d58f19ba892ae8ab5a709545..0dc81b85112c2f21b36dd0a96a95075d8fed0883 100644 --- a/drivers/acpi/acpica/utglobal.c +++ b/drivers/acpi/acpica/utglobal.c @@ -186,6 +186,10 @@ struct acpi_fixed_event_info acpi_gbl_fixed_event_info[ACPI_NUM_FIXED_EVENTS] = ACPI_BITREG_RT_CLOCK_ENABLE, ACPI_BITMASK_RT_CLOCK_STATUS, ACPI_BITMASK_RT_CLOCK_ENABLE}, + /* ACPI_EVENT_PCIE_WAKE */ {ACPI_BITREG_PCIEXP_WAKE_STATUS, + ACPI_BITREG_PCIEXP_WAKE_DISABLE, + ACPI_BITMASK_PCIEXP_WAKE_STATUS, + ACPI_BITMASK_PCIEXP_WAKE_DISABLE}, }; #endif /* !ACPI_REDUCED_HARDWARE */ diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 8678e162181f4decfb49be9bd9ec49978a2910f3..160606af8b4f55c3297b7e382ff7de77683f14d8 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -99,6 +99,20 @@ static inline bool is_hest_type_generic_v2(struct ghes *ghes) return ghes->generic->header.type == ACPI_HEST_TYPE_GENERIC_ERROR_V2; } +/* + * A platform may describe one error source for the handling of synchronous + * errors (e.g. MCE or SEA), or for handling asynchronous errors (e.g. SCI + * or External Interrupt). On x86, the HEST notifications are always + * asynchronous, so only SEA on ARM is delivered as a synchronous + * notification. + */ +static inline bool is_hest_sync_notify(struct ghes *ghes) +{ + u8 notify_type = ghes->generic->notify.type; + + return notify_type == ACPI_HEST_NOTIFY_SEA; +} + /* * This driver isn't really modular, however for the time being, * continuing to use module_param is the easiest way to remain @@ -461,7 +475,7 @@ static bool ghes_do_memory_failure(u64 physical_addr, int flags) } static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, - int sev) + int sev, bool sync) { int flags = -1; int sec_sev = ghes_severity(gdata->error_severity); @@ -475,7 +489,7 @@ static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED)) flags = MF_SOFT_OFFLINE; if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE) - flags = 0; + flags = sync ? MF_ACTION_REQUIRED : 0; if (flags != -1) return ghes_do_memory_failure(mem_err->physical_addr, flags); @@ -483,9 +497,11 @@ static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, return false; } -static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, int sev) +static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, + int sev, bool sync) { struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); + int flags = sync ? MF_ACTION_REQUIRED : 0; bool queued = false; int sec_sev, i; char *p; @@ -510,7 +526,7 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, int s * and don't filter out 'corrected' error here. */ if (is_cache && has_pa) { - queued = ghes_do_memory_failure(err_info->physical_fault_addr, 0); + queued = ghes_do_memory_failure(err_info->physical_fault_addr, flags); p += err_info->length; continue; } @@ -631,6 +647,7 @@ static bool ghes_do_proc(struct ghes *ghes, const guid_t *fru_id = &guid_null; char *fru_text = ""; bool queued = false; + bool sync = is_hest_sync_notify(ghes); sev = ghes_severity(estatus->error_severity); apei_estatus_for_each_section(estatus, gdata) { @@ -648,13 +665,13 @@ static bool ghes_do_proc(struct ghes *ghes, ghes_edac_report_mem_error(sev, mem_err); arch_apei_report_mem_error(sev, mem_err); - queued = ghes_handle_memory_failure(gdata, sev); + queued = ghes_handle_memory_failure(gdata, sev, sync); } else if (guid_equal(sec_type, &CPER_SEC_PCIE)) { ghes_handle_aer(gdata); } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { - queued = ghes_handle_arm_hw_error(gdata, sev); + queued = ghes_handle_arm_hw_error(gdata, sev, sync); } else { void *err = acpi_hest_get_payload(gdata); diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile index 6ff50f4ed947107c3a4dcee9f68fce768e64e2da..66acbe77f46e5a18cdf95f711f91ebc9c80df627 100644 --- a/drivers/acpi/arm64/Makefile +++ b/drivers/acpi/arm64/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_ACPI_IORT) += iort.o obj-$(CONFIG_ACPI_GTDT) += gtdt.o +obj-y += dma.o diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c new file mode 100644 index 0000000000000000000000000000000000000000..93d796531af307d59965e7bf62a2d70a1fa13a75 --- /dev/null +++ b/drivers/acpi/arm64/dma.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include + +void acpi_arch_dma_setup(struct device *dev) +{ + int ret; + u64 end, mask; + u64 size = 0; + const struct bus_dma_region *map = NULL; + + /* + * If @dev is expected to be DMA-capable then the bus code that created + * it should have initialised its dma_mask pointer by this point. For + * now, we'll continue the legacy behaviour of coercing it to the + * coherent mask if not, but we'll no longer do so quietly. + */ + if (!dev->dma_mask) { + dev_warn(dev, "DMA mask not set\n"); + dev->dma_mask = &dev->coherent_dma_mask; + } + + if (dev->coherent_dma_mask) + size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); + else + size = 1ULL << 32; + + ret = acpi_dma_get_range(dev, &map); + if (!ret && map) { + const struct bus_dma_region *r = map; + + for (end = 0; r->size; r++) { + if (r->dma_start + r->size - 1 > end) + end = r->dma_start + r->size - 1; + } + + size = end + 1; + dev->dma_range_map = map; + } + + if (ret == -ENODEV) + ret = iort_dma_get_ranges(dev, &size); + if (!ret) { + /* + * Limit coherent and dma mask based on size retrieved from + * firmware. + */ + end = size - 1; + mask = DMA_BIT_MASK(ilog2(end) + 1); + dev->bus_dma_limit = end; + dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); + *dev->dma_mask = min(*dev->dma_mask, mask); + } +} diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 554943be2698478adbab58915a1fa6e4b17572d1..5539779d78cdc303c1ca528b127a09d68f1aa911 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1143,56 +1143,18 @@ static int rc_dma_get_range(struct device *dev, u64 *size) } /** - * iort_dma_setup() - Set-up device DMA parameters. + * iort_dma_get_ranges() - Look up DMA addressing limit for the device + * @dev: device to lookup + * @size: DMA range size result pointer * - * @dev: device to configure - * @dma_addr: device DMA address result pointer - * @dma_size: DMA range size result pointer + * Return: 0 on success, an error otherwise. */ -void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) +int iort_dma_get_ranges(struct device *dev, u64 *size) { - u64 end, mask, dmaaddr = 0, size = 0, offset = 0; - int ret; - - /* - * If @dev is expected to be DMA-capable then the bus code that created - * it should have initialised its dma_mask pointer by this point. For - * now, we'll continue the legacy behaviour of coercing it to the - * coherent mask if not, but we'll no longer do so quietly. - */ - if (!dev->dma_mask) { - dev_warn(dev, "DMA mask not set\n"); - dev->dma_mask = &dev->coherent_dma_mask; - } - - if (dev->coherent_dma_mask) - size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); + if (dev_is_pci(dev)) + return rc_dma_get_range(dev, size); else - size = 1ULL << 32; - - ret = acpi_dma_get_range(dev, &dmaaddr, &offset, &size); - if (ret == -ENODEV) - ret = dev_is_pci(dev) ? rc_dma_get_range(dev, &size) - : nc_dma_get_range(dev, &size); - - if (!ret) { - /* - * Limit coherent and dma mask based on size retrieved from - * firmware. - */ - end = dmaaddr + size - 1; - mask = DMA_BIT_MASK(ilog2(end) + 1); - dev->bus_dma_limit = end; - dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); - *dev->dma_mask = min(*dev->dma_mask, mask); - } - - *dma_addr = dmaaddr; - *dma_size = size; - - ret = dma_direct_set_offset(dev, dmaaddr + offset, dmaaddr, size); - - dev_dbg(dev, "dma_offset(%#08llx)%s\n", offset, ret ? " failed!" : ""); + return nc_dma_get_range(dev, size); } static void __init acpi_iort_register_irq(int hwirq, const char *name, diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 60dfe63301d008aba4b2644928b2a39240ec71f4..937464e329f590fde9aa5d15fb03c4154d95b6fd 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -994,6 +994,9 @@ static int __init acpi_bus_init_irq(void) case ACPI_IRQ_MODEL_PLATFORM: message = "platform specific model"; break; + case ACPI_IRQ_MODEL_LPIC: + message = "LPIC"; + break; default: printk(KERN_WARNING PREFIX "Unknown interrupt routing model\n"); return -ENODEV; diff --git a/drivers/acpi/numa/Kconfig b/drivers/acpi/numa/Kconfig index fcf2e556d69d21477466f610408f668eb8303442..39b1f34c21dfd7a86ddc360857897a94dc0d2046 100644 --- a/drivers/acpi/numa/Kconfig +++ b/drivers/acpi/numa/Kconfig @@ -2,7 +2,7 @@ config ACPI_NUMA bool "NUMA support" depends on NUMA - depends on (X86 || IA64 || ARM64) + depends on (X86 || IA64 || ARM64 || LOONGARCH) default y if IA64 || ARM64 config ACPI_HMAT diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 6021a101344222fc74bfc5d056061337e05f9646..b8795fc49097539ba54e73d9b37d6ef9c467f034 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -206,7 +206,7 @@ int __init srat_disabled(void) return acpi_numa < 0; } -#if defined(CONFIG_X86) || defined(CONFIG_ARM64) +#if defined(CONFIG_X86) || defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH) /* * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for * I/O localities since SRAT does not list them. I/O localities are diff --git a/drivers/acpi/pci_irq.c b/drivers/acpi/pci_irq.c index 14ee631cb7cf121b673829120bb221ee369d8b6f..b3e4680003f564bbef78948ef61887d08f5a3284 100644 --- a/drivers/acpi/pci_irq.c +++ b/drivers/acpi/pci_irq.c @@ -405,7 +405,8 @@ int acpi_pci_irq_enable(struct pci_dev *dev) * controller and must therefore be considered active high * as default. */ - int polarity = acpi_irq_model == ACPI_IRQ_MODEL_GIC ? + int polarity = (acpi_irq_model == ACPI_IRQ_MODEL_GIC || + acpi_irq_model == ACPI_IRQ_MODEL_LPIC) ? ACPI_ACTIVE_HIGH : ACPI_ACTIVE_LOW; char *link = NULL; char link_desc[16]; diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 2709ef2b0351bb9677968e8f291b92095a779a5f..09c3b24938fe342334458fc2bf56e6a41526b41b 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -165,6 +165,16 @@ static struct mcfg_fixup mcfg_quirks[] = { ALTRA_ECAM_QUIRK(1, 14), ALTRA_ECAM_QUIRK(1, 15), #endif /* ARM64 */ + +#ifdef CONFIG_LOONGARCH +#define LOONGSON_ECAM_MCFG(table_id, seg) \ + { "LOONGS", table_id, 1, seg, MCFG_BUS_ANY, &loongson_pci_ecam_ops } + + LOONGSON_ECAM_MCFG("\0", 0), + LOONGSON_ECAM_MCFG("LOONGSON", 0), + LOONGSON_ECAM_MCFG("\0", 1), + LOONGSON_ECAM_MCFG("LOONGSON", 1), +#endif /* LOONGARCH */ }; static char mcfg_oem_id[ACPI_OEM_ID_SIZE]; diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 80e92c298055d775e7d38275b95785988607ee91..cf872dc5b07a64cae71525f1229eaf554544d99b 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -639,6 +639,7 @@ acpi_fwnode_get_named_child_node(const struct fwnode_handle *fwnode, * @index: Index of the reference to return * @num_args: Maximum number of arguments after each reference * @args: Location to store the returned reference with optional arguments + * (may be NULL) * * Find property with @name, verifify that it is a package containing at least * one object reference and if so, store the ACPI device object pointer to the @@ -697,6 +698,9 @@ int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode, if (ret) return ret == -ENODEV ? -EINVAL : ret; + if (!args) + return 0; + args->fwnode = acpi_fwnode_handle(device); args->nargs = 0; return 0; diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 67a5ee2fedfd3daf9305c3a028b2793703c13790..6c08fb35477300425253fd0608b920a604ca003b 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "internal.h" @@ -1411,25 +1412,21 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev) * acpi_dma_get_range() - Get device DMA parameters. * * @dev: device to configure - * @dma_addr: pointer device DMA address result - * @offset: pointer to the DMA offset result - * @size: pointer to DMA range size result + * @map: pointer to DMA ranges result * - * Evaluate DMA regions and return respectively DMA region start, offset - * and size in dma_addr, offset and size on parsing success; it does not - * update the passed in values on failure. + * Evaluate DMA regions and return pointer to DMA regions on + * parsing success; it does not update the passed in values on failure. * * Return 0 on success, < 0 on failure. */ -int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, - u64 *size) +int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map) { struct acpi_device *adev; LIST_HEAD(list); struct resource_entry *rentry; int ret; struct device *dma_dev = dev; - u64 len, dma_start = U64_MAX, dma_end = 0, dma_offset = 0; + struct bus_dma_region *r; /* * Walk the device tree chasing an ACPI companion with a _DMA @@ -1454,31 +1451,28 @@ int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, ret = acpi_dev_get_dma_resources(adev, &list); if (ret > 0) { + r = kcalloc(ret + 1, sizeof(*r), GFP_KERNEL); + if (!r) { + ret = -ENOMEM; + goto out; + } + list_for_each_entry(rentry, &list, node) { - if (dma_offset && rentry->offset != dma_offset) { + if (rentry->res->start >= rentry->res->end) { + kfree(r); ret = -EINVAL; - dev_warn(dma_dev, "Can't handle multiple windows with different offsets\n"); + dev_dbg(dma_dev, "Invalid DMA regions configuration\n"); goto out; } - dma_offset = rentry->offset; - /* Take lower and upper limits */ - if (rentry->res->start < dma_start) - dma_start = rentry->res->start; - if (rentry->res->end > dma_end) - dma_end = rentry->res->end; - } - - if (dma_start >= dma_end) { - ret = -EINVAL; - dev_dbg(dma_dev, "Invalid DMA regions configuration\n"); - goto out; + r->cpu_start = rentry->res->start; + r->dma_start = rentry->res->start - rentry->offset; + r->size = resource_size(rentry->res); + r->offset = rentry->offset; + r++; } - *dma_addr = dma_start - dma_offset; - len = dma_end - dma_start; - *size = max(len, len + 1); - *offset = dma_offset; + *map = r; } out: acpi_dev_free_resource_list(&list); @@ -1496,20 +1490,19 @@ int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, const u32 *input_id) { const struct iommu_ops *iommu; - u64 dma_addr = 0, size = 0; if (attr == DEV_DMA_NOT_SUPPORTED) { set_dma_ops(dev, &dma_dummy_ops); return 0; } - iort_dma_setup(dev, &dma_addr, &size); + acpi_arch_dma_setup(dev); iommu = iort_iommu_configure_id(dev, input_id); if (PTR_ERR(iommu) == -EPROBE_DEFER) return -EPROBE_DEFER; - arch_setup_dma_ops(dev, dma_addr, size, + arch_setup_dma_ops(dev, 0, U64_MAX, iommu, attr == DEV_DMA_COHERENT); return 0; diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 9d581045acff079276ed81a307b60ec7244f816b..b36d9a22a07a97406961abc6712fe39045c64dd5 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -207,6 +207,16 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) } break; + case ACPI_MADT_TYPE_CORE_PIC: + { + struct acpi_madt_core_pic *p = + (struct acpi_madt_core_pic *)header; + pr_debug("CORE PIC (processor_id[0x%02x] core_id[0x%02x] %s)\n", + p->processor_id, p->core_id, + (p->flags & ACPI_MADT_ENABLED) ? "enabled" : "disabled"); + } + break; + default: pr_warn("Found unsupported MADT entry (type = 0x%x)\n", header->type); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 12db190ff72b0017b80bd72ac5087a5a4c72c91c..b655bc3956c797e6898d6afdf9b5f5b2ee618c1f 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -359,8 +359,7 @@ static void debug_low_async_space_locked(struct binder_alloc *alloc, int pid) continue; if (!buffer->async_transaction) continue; - total_alloc_size += binder_alloc_buffer_size(alloc, buffer) - + sizeof(struct binder_buffer); + total_alloc_size += binder_alloc_buffer_size(alloc, buffer); num_buffers++; } @@ -415,17 +414,17 @@ static struct binder_buffer *binder_alloc_new_buf_locked( alloc->pid, extra_buffers_size); return ERR_PTR(-EINVAL); } - if (is_async && - alloc->free_async_space < size + sizeof(struct binder_buffer)) { + + /* Pad 0-size buffers so they get assigned unique addresses */ + size = max(size, sizeof(void *)); + + if (is_async && alloc->free_async_space < size) { binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, "%d: binder_alloc_buf size %zd failed, no async space left\n", alloc->pid, size); return ERR_PTR(-ENOSPC); } - /* Pad 0-size buffers so they get assigned unique addresses */ - size = max(size, sizeof(void *)); - while (n) { buffer = rb_entry(n, struct binder_buffer, rb_node); BUG_ON(!buffer->free); @@ -526,7 +525,7 @@ static struct binder_buffer *binder_alloc_new_buf_locked( buffer->extra_buffers_size = extra_buffers_size; buffer->pid = pid; if (is_async) { - alloc->free_async_space -= size + sizeof(struct binder_buffer); + alloc->free_async_space -= size; binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, "%d: binder_alloc_buf size %zd async free %zd\n", alloc->pid, size, alloc->free_async_space); @@ -662,8 +661,7 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size); if (buffer->async_transaction) { - alloc->free_async_space += buffer_size + sizeof(struct binder_buffer); - + alloc->free_async_space += buffer_size; binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, "%d: binder_free_buf size %zd async free %zd\n", alloc->pid, size, alloc->free_async_space); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 702b8e061b36e1cd4a4614ce7973c489ce2c6c02..3717ed6fcccc214a9c4e61cf40dc2d8c0b0e8708 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -5420,8 +5420,10 @@ struct ata_host *ata_host_alloc(struct device *dev, int max_ports) if (!host) return NULL; - if (!devres_open_group(dev, NULL, GFP_KERNEL)) - goto err_free; + if (!devres_open_group(dev, NULL, GFP_KERNEL)) { + kfree(host); + return NULL; + } dr = devres_alloc(ata_devres_release, 0, GFP_KERNEL); if (!dr) @@ -5453,8 +5455,6 @@ struct ata_host *ata_host_alloc(struct device *dev, int max_ports) err_out: devres_release_group(dev, NULL); - err_free: - kfree(host); return NULL; } EXPORT_SYMBOL_GPL(ata_host_alloc); diff --git a/drivers/base/core.c b/drivers/base/core.c index d98cab88c38af9d09af45c822179121d2b5b0ced..874f49cca4af1f7e12736b2a18c792adb3344b09 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1993,8 +1993,11 @@ static ssize_t uevent_show(struct device *dev, struct device_attribute *attr, if (!env) return -ENOMEM; + /* Synchronize with really_probe() */ + device_lock(dev); /* let the kset specific function add its keys */ retval = kset->uevent_ops->uevent(kset, &dev->kobj, env); + device_unlock(dev); if (retval) goto out; diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index c4a17e5edf8b4333b9e8df0977c3c450aaf9797e..1dc4c59899d4847a7bb608fb6236deb4033e1eef 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -344,7 +344,7 @@ __platform_msi_create_device_domain(struct device *dev, goto free_priv; err = msi_domain_prepare_irqs(domain->parent, dev, nvec, &data->arg); - if (err) + if (err < 0) goto free_domain; return domain; diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index d2fb3eb5816c3518aa0ad4ad1af666230a4f7bc5..b664c36388e24ff7a5fdff82647bff0371746c08 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -531,6 +531,9 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (nargs > NR_FWNODE_REFERENCE_ARGS) return -EINVAL; + if (!args) + return 0; + args->fwnode = software_node_get(refnode); args->nargs = nargs; diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index b0f7930524ba00c008ac1717a095a695726913e5..5b102d333a4104deaa1c6d446e3faca83ce6c30b 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -3517,14 +3517,15 @@ static bool rbd_lock_add_request(struct rbd_img_request *img_req) static void rbd_lock_del_request(struct rbd_img_request *img_req) { struct rbd_device *rbd_dev = img_req->rbd_dev; - bool need_wakeup; + bool need_wakeup = false; lockdep_assert_held(&rbd_dev->lock_rwsem); spin_lock(&rbd_dev->lock_lists_lock); - rbd_assert(!list_empty(&img_req->lock_item)); - list_del_init(&img_req->lock_item); - need_wakeup = (rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING && - list_empty(&rbd_dev->running_list)); + if (!list_empty(&img_req->lock_item)) { + list_del_init(&img_req->lock_item); + need_wakeup = (rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING && + list_empty(&rbd_dev->running_list)); + } spin_unlock(&rbd_dev->lock_lists_lock); if (need_wakeup) complete(&rbd_dev->releasing_wait); @@ -3907,14 +3908,19 @@ static void wake_lock_waiters(struct rbd_device *rbd_dev, int result) return; } - list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) { + while (!list_empty(&rbd_dev->acquiring_list)) { + img_req = list_first_entry(&rbd_dev->acquiring_list, + struct rbd_img_request, lock_item); mutex_lock(&img_req->state_mutex); rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK); + if (!result) + list_move_tail(&img_req->lock_item, + &rbd_dev->running_list); + else + list_del_init(&img_req->lock_item); rbd_img_schedule(img_req, result); mutex_unlock(&img_req->state_mutex); } - - list_splice_tail_init(&rbd_dev->acquiring_list, &rbd_dev->running_list); } static bool locker_equal(const struct ceph_locker *lhs, diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index 614dd287cb4ff5cf068f60092a4a26597405a9cf..49c0f5ad0b73f36e5609b7926e2975c712162abe 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -569,6 +569,8 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, mhi_del_ring_element(mhi_cntrl, tre_ring); local_rp = tre_ring->rp; + read_unlock_bh(&mhi_chan->lock); + /* notify client */ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); @@ -591,6 +593,8 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, kfree(buf_info->cb_buf); } } + + read_lock_bh(&mhi_chan->lock); } break; } /* CC_EOT */ diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index b89f300751b1b5d911197d6955ae718f31a9c06d..e955ed7497c0b374b537fe256166383624956f61 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -749,7 +749,8 @@ int ipmi_smi_watcher_register(struct ipmi_smi_watcher *watcher) list_add(&watcher->link, &smi_watchers); index = srcu_read_lock(&ipmi_interfaces_srcu); - list_for_each_entry_rcu(intf, &ipmi_interfaces, link) { + list_for_each_entry_rcu(intf, &ipmi_interfaces, link, + lockdep_is_held(&smi_watchers_mutex)) { int intf_num = READ_ONCE(intf->intf_num); if (intf_num == -1) diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c index 45501637705c30f88317e1cf70bba8d4dcc92cfd..62e994d18fe249bf32bf183827c1b34638823d47 100644 --- a/drivers/clk/clk-fixed-rate.c +++ b/drivers/clk/clk-fixed-rate.c @@ -49,12 +49,24 @@ const struct clk_ops clk_fixed_rate_ops = { }; EXPORT_SYMBOL_GPL(clk_fixed_rate_ops); +static void devm_clk_hw_register_fixed_rate_release(struct device *dev, void *res) +{ + struct clk_fixed_rate *fix = res; + + /* + * We can not use clk_hw_unregister_fixed_rate, since it will kfree() + * the hw, resulting in double free. Just unregister the hw and let + * devres code kfree() it. + */ + clk_hw_unregister(&fix->hw); +} + struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, struct device_node *np, const char *name, const char *parent_name, const struct clk_hw *parent_hw, const struct clk_parent_data *parent_data, unsigned long flags, unsigned long fixed_rate, unsigned long fixed_accuracy, - unsigned long clk_fixed_flags) + unsigned long clk_fixed_flags, bool devm) { struct clk_fixed_rate *fixed; struct clk_hw *hw; @@ -62,7 +74,11 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, int ret = -EINVAL; /* allocate fixed-rate clock */ - fixed = kzalloc(sizeof(*fixed), GFP_KERNEL); + if (devm) + fixed = devres_alloc(devm_clk_hw_register_fixed_rate_release, + sizeof(*fixed), GFP_KERNEL); + else + fixed = kzalloc(sizeof(*fixed), GFP_KERNEL); if (!fixed) return ERR_PTR(-ENOMEM); @@ -90,9 +106,13 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, else if (np) ret = of_clk_hw_register(np, hw); if (ret) { - kfree(fixed); + if (devm) + devres_free(fixed); + else + kfree(fixed); hw = ERR_PTR(ret); - } + } else if (devm) + devres_add(dev, fixed); return hw; } diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c index 4dea29fa901d49db0a087feabec2bc4aa817be90..d6515661580c19b4ed0376e0801adf1d366ff21a 100644 --- a/drivers/clk/clk-si5341.c +++ b/drivers/clk/clk-si5341.c @@ -888,10 +888,8 @@ static int si5341_output_clk_set_rate(struct clk_hw *hw, unsigned long rate, r[0] = r_div ? (r_div & 0xff) : 1; r[1] = (r_div >> 8) & 0xff; r[2] = (r_div >> 16) & 0xff; - err = regmap_bulk_write(output->data->regmap, + return regmap_bulk_write(output->data->regmap, SI5341_OUT_R_REG(output), r, 3); - - return 0; } static int si5341_output_reparent(struct clk_si5341_output *output, u8 index) diff --git a/drivers/clk/qcom/gpucc-sm8150.c b/drivers/clk/qcom/gpucc-sm8150.c index 27c40754b2c79e41fc985b779957370e30ee49ea..3d9b296a6e4a8d63c58477418520efb19f772475 100644 --- a/drivers/clk/qcom/gpucc-sm8150.c +++ b/drivers/clk/qcom/gpucc-sm8150.c @@ -38,8 +38,8 @@ static struct alpha_pll_config gpu_cc_pll1_config = { .config_ctl_hi_val = 0x00002267, .config_ctl_hi1_val = 0x00000024, .test_ctl_val = 0x00000000, - .test_ctl_hi_val = 0x00000002, - .test_ctl_hi1_val = 0x00000000, + .test_ctl_hi_val = 0x00000000, + .test_ctl_hi1_val = 0x00000020, .user_ctl_val = 0x00000000, .user_ctl_hi_val = 0x00000805, .user_ctl_hi1_val = 0x000000d0, diff --git a/drivers/clk/qcom/mmcc-apq8084.c b/drivers/clk/qcom/mmcc-apq8084.c index fbfcf000673947fd61ebf5d467ce61ea5f2275da..c2fd0e8f4bc096e6dd106ca93b142c87f057e79f 100644 --- a/drivers/clk/qcom/mmcc-apq8084.c +++ b/drivers/clk/qcom/mmcc-apq8084.c @@ -333,6 +333,7 @@ static struct freq_tbl ftbl_mmss_axi_clk[] = { F(333430000, P_MMPLL1, 3.5, 0, 0), F(400000000, P_MMPLL0, 2, 0, 0), F(466800000, P_MMPLL1, 2.5, 0, 0), + { } }; static struct clk_rcg2 mmss_axi_clk_src = { @@ -357,6 +358,7 @@ static struct freq_tbl ftbl_ocmemnoc_clk[] = { F(150000000, P_GPLL0, 4, 0, 0), F(228570000, P_MMPLL0, 3.5, 0, 0), F(320000000, P_MMPLL0, 2.5, 0, 0), + { } }; static struct clk_rcg2 ocmemnoc_clk_src = { diff --git a/drivers/clk/qcom/videocc-sm8150.c b/drivers/clk/qcom/videocc-sm8150.c index 3087e2ec8fd47ec3de36b3380ab6ae8903f706be..61089cde4222ec3bb28994f166d7fb1d94a9598b 100644 --- a/drivers/clk/qcom/videocc-sm8150.c +++ b/drivers/clk/qcom/videocc-sm8150.c @@ -37,6 +37,7 @@ static struct alpha_pll_config video_pll0_config = { .config_ctl_val = 0x20485699, .config_ctl_hi_val = 0x00002267, .config_ctl_hi1_val = 0x00000024, + .test_ctl_hi1_val = 0x00000020, .user_ctl_val = 0x00000000, .user_ctl_hi_val = 0x00000805, .user_ctl_hi1_val = 0x000000D0, @@ -218,6 +219,10 @@ static const struct regmap_config video_cc_sm8150_regmap_config = { static const struct qcom_reset_map video_cc_sm8150_resets[] = { [VIDEO_CC_MVSC_CORE_CLK_BCR] = { 0x850, 2 }, + [VIDEO_CC_INTERFACE_BCR] = { 0x8f0 }, + [VIDEO_CC_MVS0_BCR] = { 0x870 }, + [VIDEO_CC_MVS1_BCR] = { 0x8b0 }, + [VIDEO_CC_MVSC_BCR] = { 0x810 }, }; static const struct qcom_cc_desc video_cc_sm8150_desc = { diff --git a/drivers/clk/zynqmp/clk-mux-zynqmp.c b/drivers/clk/zynqmp/clk-mux-zynqmp.c index 06194149be831208f4bd5321ab30b01a27b40998..46ff5cb733ee115a1944b9085e81e3739fb5b52a 100644 --- a/drivers/clk/zynqmp/clk-mux-zynqmp.c +++ b/drivers/clk/zynqmp/clk-mux-zynqmp.c @@ -83,7 +83,7 @@ static int zynqmp_clk_mux_set_parent(struct clk_hw *hw, u8 index) static const struct clk_ops zynqmp_clk_mux_ops = { .get_parent = zynqmp_clk_mux_get_parent, .set_parent = zynqmp_clk_mux_set_parent, - .determine_rate = __clk_mux_determine_rate, + .determine_rate = __clk_mux_determine_rate_closest, }; static const struct clk_ops zynqmp_clk_mux_ro_ops = { diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index 66da02b83d3936246130fd924169be23e6816363..acfd4878cce20e3b76c16cd1028e4f61052f1ff6 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -109,49 +109,6 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, return DIV_ROUND_UP_ULL(parent_rate, value); } -static void zynqmp_get_divider2_val(struct clk_hw *hw, - unsigned long rate, - struct zynqmp_clk_divider *divider, - int *bestdiv) -{ - int div1; - int div2; - long error = LONG_MAX; - unsigned long div1_prate; - struct clk_hw *div1_parent_hw; - struct clk_hw *div2_parent_hw = clk_hw_get_parent(hw); - struct zynqmp_clk_divider *pdivider = - to_zynqmp_clk_divider(div2_parent_hw); - - if (!pdivider) - return; - - div1_parent_hw = clk_hw_get_parent(div2_parent_hw); - if (!div1_parent_hw) - return; - - div1_prate = clk_hw_get_rate(div1_parent_hw); - *bestdiv = 1; - for (div1 = 1; div1 <= pdivider->max_div;) { - for (div2 = 1; div2 <= divider->max_div;) { - long new_error = ((div1_prate / div1) / div2) - rate; - - if (abs(new_error) < abs(error)) { - *bestdiv = div2; - error = new_error; - } - if (divider->flags & CLK_DIVIDER_POWER_OF_TWO) - div2 = div2 << 1; - else - div2++; - } - if (pdivider->flags & CLK_DIVIDER_POWER_OF_TWO) - div1 = div1 << 1; - else - div1++; - } -} - /** * zynqmp_clk_divider_round_rate() - Round rate of divider clock * @hw: handle between common and hardware-specific interfaces @@ -170,6 +127,7 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw, u32 div_type = divider->div_type; u32 bestdiv; int ret; + u8 width; /* if read only, just return current value */ if (divider->flags & CLK_DIVIDER_READ_ONLY) { @@ -189,23 +147,12 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw, return DIV_ROUND_UP_ULL((u64)*prate, bestdiv); } - bestdiv = zynqmp_divider_get_val(*prate, rate, divider->flags); - - /* - * In case of two divisors, compute best divider values and return - * divider2 value based on compute value. div1 will be automatically - * set to optimum based on required total divider value. - */ - if (div_type == TYPE_DIV2 && - (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) { - zynqmp_get_divider2_val(hw, rate, divider, &bestdiv); - } + width = fls(divider->max_div); - if ((clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && divider->is_frac) - bestdiv = rate % *prate ? 1 : bestdiv; + rate = divider_round_rate(hw, rate, prate, NULL, width, divider->flags); - bestdiv = min_t(u32, bestdiv, divider->max_div); - *prate = rate * bestdiv; + if (divider->is_frac && (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && (rate % *prate)) + *prate = rate; return rate; } diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 85de313ddec29cdc33824ec98964abe2bae8c962..fdbe7ceb157919f2e265a99ab7ebf6007a37d4c8 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -287,6 +287,20 @@ config LOONGSON1_CPUFREQ If in doubt, say N. endif +if LOONGARCH +config LOONGSON3_CPUFREQ + tristate "Loongson3 CPUFreq Driver" + help + This option adds a CPUFreq driver for Loongson processors which + support software configurable cpu frequency. + + Loongson-3 family processors support this feature. + + For details, take a look at . + + If in doubt, say N. +endif + if SPARC64 config SPARC_US3_CPUFREQ tristate "UltraSPARC-III CPU Frequency driver" diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index f1b7e3dd6e5daf38c249f15ec2aeb2baa636369c..cf2d6bc976e615180259c4bdf196dfa2f36e611e 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -105,6 +105,7 @@ obj-$(CONFIG_POWERNV_CPUFREQ) += powernv-cpufreq.o obj-$(CONFIG_BMIPS_CPUFREQ) += bmips-cpufreq.o obj-$(CONFIG_IA64_ACPI_CPUFREQ) += ia64-acpi-cpufreq.o obj-$(CONFIG_LOONGSON2_CPUFREQ) += loongson2_cpufreq.o +obj-$(CONFIG_LOONGSON3_CPUFREQ) += loongson3_cpufreq.o obj-$(CONFIG_LOONGSON1_CPUFREQ) += loongson1-cpufreq.o obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 5b4bca71f201dc846b3b56e209e3592aea4c167a..162de402b1134031d5b3fe3a5e7b6e5ad9ccd9eb 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1559,47 +1559,36 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) return 0; } -static int cpufreq_offline(unsigned int cpu) +static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) { - struct cpufreq_policy *policy; int ret; - pr_debug("%s: unregistering CPU %u\n", __func__, cpu); - - policy = cpufreq_cpu_get_raw(cpu); - if (!policy) { - pr_debug("%s: No cpu_data found\n", __func__); - return 0; - } - - down_write(&policy->rwsem); if (has_target()) cpufreq_stop_governor(policy); cpumask_clear_cpu(cpu, policy->cpus); - if (policy_is_inactive(policy)) { - if (has_target()) - strncpy(policy->last_governor, policy->governor->name, - CPUFREQ_NAME_LEN); - else - policy->last_policy = policy->policy; - } else if (cpu == policy->cpu) { - /* Nominate new CPU */ - policy->cpu = cpumask_any(policy->cpus); - } - - /* Start governor again for active policy */ if (!policy_is_inactive(policy)) { + /* Nominate a new CPU if necessary. */ + if (cpu == policy->cpu) + policy->cpu = cpumask_any(policy->cpus); + + /* Start the governor again for the active policy. */ if (has_target()) { ret = cpufreq_start_governor(policy); if (ret) pr_err("%s: Failed to start governor\n", __func__); } - goto unlock; + return; } + if (has_target()) + strncpy(policy->last_governor, policy->governor->name, + CPUFREQ_NAME_LEN); + else + policy->last_policy = policy->policy; + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { cpufreq_cooling_unregister(policy->cdev); policy->cdev = NULL; @@ -1617,12 +1606,31 @@ static int cpufreq_offline(unsigned int cpu) */ if (cpufreq_driver->offline) { cpufreq_driver->offline(policy); - } else if (cpufreq_driver->exit) { + return; + } + + if (cpufreq_driver->exit) cpufreq_driver->exit(policy); - policy->freq_table = NULL; + + policy->freq_table = NULL; +} + +static int cpufreq_offline(unsigned int cpu) +{ + struct cpufreq_policy *policy; + + pr_debug("%s: unregistering CPU %u\n", __func__, cpu); + + policy = cpufreq_cpu_get_raw(cpu); + if (!policy) { + pr_debug("%s: No cpu_data found\n", __func__); + return 0; } -unlock: + down_write(&policy->rwsem); + + __cpufreq_offline(cpu, policy); + up_write(&policy->rwsem); return 0; } @@ -1640,19 +1648,26 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) if (!policy) return; + down_write(&policy->rwsem); + if (cpu_online(cpu)) - cpufreq_offline(cpu); + __cpufreq_offline(cpu, policy); cpumask_clear_cpu(cpu, policy->real_cpus); remove_cpu_dev_symlink(policy, dev); - if (cpumask_empty(policy->real_cpus)) { - /* We did light-weight exit earlier, do full tear down now */ - if (cpufreq_driver->offline) - cpufreq_driver->exit(policy); - - cpufreq_policy_free(policy); + if (!cpumask_empty(policy->real_cpus)) { + up_write(&policy->rwsem); + return; } + + /* We did light-weight exit earlier, do full tear down now */ + if (cpufreq_driver->offline && cpufreq_driver->exit) + cpufreq_driver->exit(policy); + + up_write(&policy->rwsem); + + cpufreq_policy_free(policy); } /** diff --git a/drivers/cpufreq/loongson3_cpufreq.c b/drivers/cpufreq/loongson3_cpufreq.c new file mode 100644 index 0000000000000000000000000000000000000000..ace30f95384e4c2371d4f2a6638fddfd1bba36a2 --- /dev/null +++ b/drivers/cpufreq/loongson3_cpufreq.c @@ -0,0 +1,419 @@ +/* + * CPUFreq driver for the loongson-3 processors + * + * All revisions of Loongson-3 processor support this feature. + * + * Author: Huacai Chen + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +#include +#include +#include + +/* Message */ +union smc_message +{ + u32 value; + struct + { + u32 id : 4; + u32 info : 4; + u32 val : 16; + u32 cmd : 6; + u32 extra : 1; + u32 complete : 1; + }; +}; + +/* Command return values */ +#define CMD_OK 0 /* No error */ +#define CMD_ERROR 1 /* Regular error */ +#define CMD_NOCMD 2 /* Command does not support */ +#define CMD_INVAL 3 /* Invalid Parameter */ + +/* Version commands */ +/* + * CMD_GET_VERSION - Get interface version + * Input: none + * Output: version + */ +#define CMD_GET_VERSION 0x1 + +/* Feature commands */ +/* + * CMD_GET_FEATURE - Get feature state + * Input: feature ID + * Output: feature flag + */ +#define CMD_GET_FEATURE 0x2 + +/* + * CMD_SET_FEATURE - Set feature state + * Input: feature ID, feature flag + * output: none + */ +#define CMD_SET_FEATURE 0x3 + +/* Feature IDs */ +#define FEATURE_SENSOR 0 +#define FEATURE_FAN 1 +#define FEATURE_DVFS 2 + +/* Sensor feature flags */ +#define FEATURE_SENSOR_ENABLE (1 << 0) +#define FEATURE_SENSOR_SAMPLE (1 << 1) + +/* Fan feature flags */ +#define FEATURE_FAN_ENABLE (1 << 0) +#define FEATURE_FAN_AUTO (1 << 1) + +/* DVFS feature flags */ +#define FEATURE_DVFS_ENABLE (1 << 0) +#define FEATURE_DVFS_BOOST (1 << 1) + +/* Sensor commands */ +/* + * CMD_GET_SENSOR_NUM - Get number of sensors + * Input: none + * Output: number + */ +#define CMD_GET_SENSOR_NUM 0x4 + +/* + * CMD_GET_SENSOR_STATUS - Get sensor status + * Input: sensor ID, type + * Output: sensor status + */ +#define CMD_GET_SENSOR_STATUS 0x5 + +/* Sensor types */ +#define SENSOR_INFO_TYPE 0 +#define SENSOR_INFO_TYPE_TEMP 1 + +/* Fan commands */ +/* + * CMD_GET_FAN_NUM - Get number of fans + * Input: none + * Output: number + */ +#define CMD_GET_FAN_NUM 0x6 + +/* + * CMD_GET_FAN_INFO - Get fan status + * Input: fan ID, type + * Output: fan info + */ +#define CMD_GET_FAN_INFO 0x7 + +/* + * CMD_SET_FAN_INFO - Set fan status + * Input: fan ID, type, value + * Output: none + */ +#define CMD_SET_FAN_INFO 0x8 + +/* Fan types */ +#define FAN_INFO_TYPE_LEVEL 0 + +/* DVFS commands */ +/* + * CMD_GET_FREQ_LEVEL_NUM - Get number of freq levels + * Input: none + * Output: number + */ +#define CMD_GET_FREQ_LEVEL_NUM 0x9 + +/* + * CMD_GET_FREQ_BOOST_LEVEL - Get number of boost levels + * Input: none + * Output: number + */ +#define CMD_GET_FREQ_BOOST_LEVEL 0x10 + +/* + * CMD_GET_FREQ_LEVEL_INFO - Get freq level info + * Input: level ID + * Output: level info + */ +#define CMD_GET_FREQ_LEVEL_INFO 0x11 + +/* + * CMD_GET_FREQ_INFO - Get freq info + * Input: CPU ID, type + * Output: freq info + */ +#define CMD_GET_FREQ_INFO 0x12 + +/* + * CMD_SET_FREQ_INFO - Set freq info + * Input: CPU ID, type, value + * Output: none + */ +#define CMD_SET_FREQ_INFO 0x13 + +/* Freq types */ +#define FREQ_INFO_TYPE_FREQ 0 +#define FREQ_INFO_TYPE_LEVEL 1 + +static inline int do_service_request(union smc_message *msg) +{ + int retries; + union smc_message last; + + last.value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + if (!last.complete) + return -1; + + iocsr_write32(msg->value, LOONGARCH_IOCSR_SMCMBX); + iocsr_write32(iocsr_read32(LOONGARCH_IOCSR_MISC_FUNC) | IOCSR_MISC_FUNC_SOFT_INT, + LOONGARCH_IOCSR_MISC_FUNC); + + for (retries = 0; retries < 10000; retries++) { + msg->value = iocsr_read32(LOONGARCH_IOCSR_SMCMBX); + if (msg->complete) + break; + + usleep_range(4, 8); + } + + if (!msg->complete || msg->cmd != CMD_OK) + return -1; + + return 0; +} + +static int boost_supported = 0; +static struct mutex cpufreq_mutex[MAX_PACKAGES]; + +enum freq { + FREQ_LEV0, /* Reserved */ + FREQ_LEV1, FREQ_LEV2, FREQ_LEV3, FREQ_LEV4, + FREQ_LEV5, FREQ_LEV6, FREQ_LEV7, FREQ_LEV8, + FREQ_LEV9, FREQ_LEV10, FREQ_LEV11, FREQ_LEV12, + FREQ_LEV13, FREQ_LEV14, FREQ_LEV15, FREQ_LEV16, + FREQ_RESV +}; + +/* For Loongson-3A5000, support boost */ +static struct cpufreq_frequency_table loongson3_cpufreq_table[] = { + {0, FREQ_LEV0, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV1, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV2, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV3, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV4, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV5, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV6, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV7, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV8, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV9, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV10, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV11, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV12, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV13, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV14, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV15, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_LEV16, CPUFREQ_ENTRY_INVALID}, + {0, FREQ_RESV, CPUFREQ_TABLE_END}, +}; + +static unsigned int loongson3_cpufreq_get(unsigned int cpu) +{ + union smc_message msg; + + msg.id = cpu; + msg.info = FREQ_INFO_TYPE_FREQ; + msg.cmd = CMD_GET_FREQ_INFO; + msg.extra = 0; + msg.complete = 0; + do_service_request(&msg); + + return msg.val * 1000; +} + +static int loongson3_cpufreq_set(struct cpufreq_policy *policy, int freq_level) +{ + uint32_t core_id = cpu_data[policy->cpu].core; + union smc_message msg; + + msg.id = core_id; + msg.info = FREQ_INFO_TYPE_LEVEL; + msg.val = freq_level; + msg.cmd = CMD_SET_FREQ_INFO; + msg.extra = 0; + msg.complete = 0; + do_service_request(&msg); + + return 0; +} + +/* + * Here we notify other drivers of the proposed change and the final change. + */ +static int loongson3_cpufreq_target(struct cpufreq_policy *policy, + unsigned int index) +{ + unsigned int cpu = policy->cpu; + unsigned int package = cpu_data[cpu].package; + + if (!cpu_online(cpu)) + return -ENODEV; + + /* setting the cpu frequency */ + mutex_lock(&cpufreq_mutex[package]); + loongson3_cpufreq_set(policy, index); + mutex_unlock(&cpufreq_mutex[package]); + + return 0; +} + +static int loongson3_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + if (!cpu_online(policy->cpu)) + return -ENODEV; + + policy->cur = loongson3_cpufreq_get(policy->cpu); + + policy->cpuinfo.transition_latency = 5000; + policy->freq_table = loongson3_cpufreq_table; + cpumask_copy(policy->cpus, topology_sibling_cpumask(policy->cpu)); + + return 0; +} + +static int loongson3_cpufreq_exit(struct cpufreq_policy *policy) +{ + return 0; +} + +static struct cpufreq_driver loongson3_cpufreq_driver = { + .name = "loongson3", + .flags = CPUFREQ_CONST_LOOPS, + .init = loongson3_cpufreq_cpu_init, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = loongson3_cpufreq_target, + .get = loongson3_cpufreq_get, + .exit = loongson3_cpufreq_exit, + .attr = cpufreq_generic_attr, +}; + +static struct platform_device_id cpufreq_id_table[] = { + { "loongson3_cpufreq", }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(platform, cpufreq_id_table); + +static struct platform_driver cpufreq_driver = { + .driver = { + .name = "loongson3_cpufreq", + .owner = THIS_MODULE, + }, + .id_table = cpufreq_id_table, +}; + +static int configure_cpufreq_info(void) +{ + int i, r, boost_level, max_level; + union smc_message msg; + + if (!cpu_has_csr) + return -EPERM; + + msg.cmd = CMD_GET_VERSION; + msg.extra = 0; + msg.complete = 0; + r = do_service_request(&msg); + if (r < 0 || msg.val < 0x1) + return -EPERM; + + msg.id = FEATURE_DVFS; + msg.val = FEATURE_DVFS_ENABLE | FEATURE_DVFS_BOOST; + msg.cmd = CMD_SET_FEATURE; + msg.extra = 0; + msg.complete = 0; + r = do_service_request(&msg); + if (r < 0) + return -EPERM; + + msg.cmd = CMD_GET_FREQ_LEVEL_NUM; + msg.extra = 0; + msg.complete = 0; + r = do_service_request(&msg); + if (r < 0) + return -EPERM; + max_level = msg.val; + + msg.cmd = CMD_GET_FREQ_BOOST_LEVEL; + msg.extra = 0; + msg.complete = 0; + r = do_service_request(&msg); + if (r < 0) + return -EPERM; + boost_level = msg.val; + if (boost_level < max_level) + boost_supported = 1; + + for (i = 0; i < max_level; i++) { + msg.cmd = CMD_GET_FREQ_LEVEL_INFO; + msg.id = i; + msg.info = FREQ_INFO_TYPE_FREQ; + msg.extra = 0; + msg.complete = 0; + r = do_service_request(&msg); + if (r < 0) + return -EPERM; + + loongson3_cpufreq_table[i].frequency = msg.val * 1000; + if (i >= boost_level) + loongson3_cpufreq_table[i].flags = CPUFREQ_BOOST_FREQ; + } + + return 0; +} + +static int __init cpufreq_init(void) +{ + int i, ret; + + ret = platform_driver_register(&cpufreq_driver); + if (ret) + goto err; + + ret = configure_cpufreq_info(); + if (ret) + goto err; + + for (i = 0; i < MAX_PACKAGES; i++) + mutex_init(&cpufreq_mutex[i]); + + ret = cpufreq_register_driver(&loongson3_cpufreq_driver); + + if (boost_supported) + cpufreq_enable_boost_support(); + + pr_info("cpufreq: Loongson-3 CPU frequency driver.\n"); + + return ret; + +err: + platform_driver_unregister(&cpufreq_driver); + return ret; +} + +static void __exit cpufreq_exit(void) +{ + cpufreq_unregister_driver(&loongson3_cpufreq_driver); + platform_driver_unregister(&cpufreq_driver); +} + +module_init(cpufreq_init); +module_exit(cpufreq_exit); + +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("CPUFreq driver for Loongson-3 processors"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index 90a920e7f6642ff4eb2bee37d81c28c46548eee0..c439be1650c84d1737364541ddc54ec5e5ec183a 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -104,7 +104,7 @@ static struct stm32_crc *stm32_crc_get_next_crc(void) struct stm32_crc *crc; spin_lock_bh(&crc_list.lock); - crc = list_first_entry(&crc_list.dev_list, struct stm32_crc, list); + crc = list_first_entry_or_null(&crc_list.dev_list, struct stm32_crc, list); if (crc) list_move_tail(&crc->list, &crc_list.dev_list); spin_unlock_bh(&crc_list.lock); diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 382ccec9ab1282d2d10a28e644efdce81383fd27..2cfc36d141c070bac9459db118e242a53c919a73 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -56,6 +56,10 @@ static void virtio_crypto_akcipher_finalize_req( struct virtio_crypto_akcipher_request *vc_akcipher_req, struct akcipher_request *req, int err) { + kfree(vc_akcipher_req->src_buf); + kfree(vc_akcipher_req->dst_buf); + vc_akcipher_req->src_buf = NULL; + vc_akcipher_req->dst_buf = NULL; virtcrypto_clear_request(&vc_akcipher_req->base); crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 32c73e7d82433678f7ac99a610675c2c89900a14..ae645c9d47a7a7c95185861ca0c64024a0010bf8 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -239,7 +239,8 @@ static int virtio_crypto_alg_skcipher_close_session( pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", ctrl_status->status, destroy_session->session_id); - return -EINVAL; + err = -EINVAL; + goto out; } err = 0; diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index d9895491ff34e374b1343ec29ce449404d6faa30..29dc13fee019587fac27a6fed99ddc8112bb5e63 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -106,7 +106,7 @@ config EFI_GENERIC_STUB config EFI_ARMSTUB_DTB_LOADER bool "Enable the DTB loader" - depends on EFI_GENERIC_STUB && !RISCV + depends on EFI_GENERIC_STUB && !RISCV && !LOONGARCH default y help Select this config option to add support for the dtb= command @@ -123,13 +123,16 @@ config EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER bool "Enable the command line initrd loader" if !X86 depends on EFI_STUB && (EFI_GENERIC_STUB || X86) default y - depends on !RISCV help Select this config option to add support for the initrd= command - line parameter, allowing an initrd that resides on the same volume - as the kernel image to be loaded into memory. - - This method is deprecated. + line parameter, allowing an initrd to be loaded into memory that + resides on a file system backed by an implementation of + EFI_SIMPLE_FILE_SYSTEM_PROTOCOL. + + This method has been superseded by the simpler LoadFile2 based + initrd loading method, but the initrd= loader is retained as it + can be used from the UEFI Shell or other generic loaders that + don't implement the Linux specific LoadFile2 method. config EFI_BOOTLOADER_CONTROL tristate "EFI Bootloader Control" diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 332739f3eded570b0f4baa74d8fa5b343cef2a2e..7670fab162ffcd1ac45bf485d3c70db27959c55f 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ EXPORT_SYMBOL(efi); unsigned long __ro_after_init efi_rng_seed = EFI_INVALID_TABLE_ADDR; static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR; static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR; +static unsigned long __initdata initrd = EFI_INVALID_TABLE_ADDR; struct mm_struct efi_mm = { .mm_rb = RB_ROOT, @@ -522,6 +524,7 @@ static const efi_config_table_type_t common_tables[] __initconst = { {LINUX_EFI_TPM_EVENT_LOG_GUID, &efi.tpm_log, "TPMEventLog" }, {LINUX_EFI_TPM_FINAL_LOG_GUID, &efi.tpm_final_log, "TPMFinalLog" }, {LINUX_EFI_MEMRESERVE_TABLE_GUID, &mem_reserve, "MEMRESERVE" }, + {LINUX_EFI_INITRD_MEDIA_GUID, &initrd, "INITRD" }, {EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" }, #ifdef CONFIG_EFI_RCI2_TABLE {DELLEMC_EFI_RCI2_TABLE_GUID, &rci2_table_phys }, @@ -661,6 +664,18 @@ int __init efi_config_parse_tables(const efi_config_table_t *config_tables, } } + if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && + initrd != EFI_INVALID_TABLE_ADDR && phys_initrd_size == 0) { + struct linux_efi_initrd *tbl; + + tbl = early_memremap(initrd, sizeof(*tbl)); + if (tbl) { + phys_initrd_start = tbl->base; + phys_initrd_size = tbl->size; + early_memunmap(tbl, sizeof(*tbl)); + } + } + return 0; } diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 2c67f71f23753784d63d4443d916a878040e2368..6c4be42170c5e398964e3e29a4d3b5d835ab1321 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -26,8 +26,10 @@ cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ $(call cc-option,-mno-single-pic-base) cflags-$(CONFIG_RISCV) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ -fpic +cflags-$(CONFIG_LOONGARCH) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ + -fpie -cflags-$(CONFIG_EFI_GENERIC_STUB) += -I$(srctree)/scripts/dtc/libfdt +cflags-$(CONFIG_EFI_PARAMS_FROM_FDT) += -I$(srctree)/scripts/dtc/libfdt KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ -include $(srctree)/include/linux/hidden.h \ @@ -64,19 +66,24 @@ lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ alignedmem.o relocate.o vsprintf.o -# include the stub's generic dependencies from lib/ when building for ARM/arm64 -efi-deps-y := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c fdt_empty_tree.c fdt_sw.c +# include the stub's libfdt dependencies from lib/ when needed +libfdt-deps := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c \ + fdt_empty_tree.c fdt_sw.c + +lib-$(CONFIG_EFI_PARAMS_FROM_FDT) += fdt.o \ + $(patsubst %.c,lib-%.o,$(libfdt-deps)) $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) -lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o fdt.o string.o \ - $(patsubst %.c,lib-%.o,$(efi-deps-y)) +lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += arm64-stub.o lib-$(CONFIG_X86) += x86-stub.o lib-$(CONFIG_RISCV) += riscv-stub.o +lib-$(CONFIG_LOONGARCH) += loongarch-stub.o + CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) # Even when -mbranch-protection=none is set, Clang will generate a @@ -104,6 +111,7 @@ STUBCOPY_FLAGS-$(CONFIG_ARM) += --rename-section .data=.data.efistub \ --rename-section .bss=.bss.efistub,load,alloc STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS + # # arm64 puts the stub in the kernel proper, which will unnecessarily retain all # code indefinitely unless it is annotated as __init/__initdata/__initconst etc. @@ -132,6 +140,11 @@ STUBCOPY_FLAGS-$(CONFIG_RISCV) += --prefix-alloc-sections=.init \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_RISCV) := R_RISCV_HI20 +# For LoongArch, keep all the symbols in .init section and make sure that no +# absolute symbols references doesn't exist. +STUBCOPY_FLAGS-$(CONFIG_LOONGARCH) += --prefix-alloc-sections=.init +STUBCOPY_RELOC-$(CONFIG_LOONGARCH) := R_LARCH_MARK_LA + $(obj)/%.stub.o: $(obj)/%.o FORCE $(call if_changed,stubcopy) diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 7f4bafcd9d335b65d855bbd34a298699608b34eb..0e721c069781ff2040168f6766a9429c75ca1bc9 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -42,26 +42,17 @@ efi_status_t check_platform_features(void) */ static bool check_image_region(u64 base, u64 size) { - unsigned long map_size, desc_size, buff_size; - efi_memory_desc_t *memory_map; - struct efi_boot_memmap map; + struct efi_boot_memmap *map; efi_status_t status; bool ret = false; int map_offset; - map.map = &memory_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = NULL; - map.key_ptr = NULL; - map.buff_size = &buff_size; - - status = efi_get_memory_map(&map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) return false; - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE; /* @@ -74,7 +65,7 @@ static bool check_image_region(u64 base, u64 size) } } - efi_bs_call(free_pool, memory_map); + efi_bs_call(free_pool, map); return ret; } diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c index aa8da0a4982941958faef955582c505815a952e5..210f6f8bfa6ac357ea95d32e6ac1ee4ef95b73a7 100644 --- a/drivers/firmware/efi/libstub/efi-stub-helper.c +++ b/drivers/firmware/efi/libstub/efi-stub-helper.c @@ -419,7 +419,6 @@ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) /** * efi_exit_boot_services() - Exit boot services * @handle: handle of the exiting image - * @map: pointer to receive the memory map * @priv: argument to be passed to @priv_func * @priv_func: function to process the memory map before exiting boot services * @@ -432,26 +431,26 @@ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) * * Return: status code */ -efi_status_t efi_exit_boot_services(void *handle, - struct efi_boot_memmap *map, - void *priv, +efi_status_t efi_exit_boot_services(void *handle, void *priv, efi_exit_boot_map_processing priv_func) { + struct efi_boot_memmap *map; efi_status_t status; - status = efi_get_memory_map(map); - + status = efi_get_memory_map(&map, true); if (status != EFI_SUCCESS) - goto fail; + return status; status = priv_func(map, priv); - if (status != EFI_SUCCESS) - goto free_map; + if (status != EFI_SUCCESS) { + efi_bs_call(free_pool, map); + return status; + } if (efi_disable_pci_dma) efi_pci_disable_bridge_busmaster(); - status = efi_bs_call(exit_boot_services, handle, *map->key_ptr); + status = efi_bs_call(exit_boot_services, handle, map->map_key); if (status == EFI_INVALID_PARAMETER) { /* @@ -467,35 +466,26 @@ efi_status_t efi_exit_boot_services(void *handle, * buffer should account for any changes in the map so the call * to get_memory_map() is expected to succeed here. */ - *map->map_size = *map->buff_size; + map->map_size = map->buff_size; status = efi_bs_call(get_memory_map, - map->map_size, - *map->map, - map->key_ptr, - map->desc_size, - map->desc_ver); + &map->map_size, + &map->map, + &map->map_key, + &map->desc_size, + &map->desc_ver); /* exit_boot_services() was called, thus cannot free */ if (status != EFI_SUCCESS) - goto fail; + return status; status = priv_func(map, priv); /* exit_boot_services() was called, thus cannot free */ if (status != EFI_SUCCESS) - goto fail; + return status; - status = efi_bs_call(exit_boot_services, handle, *map->key_ptr); + status = efi_bs_call(exit_boot_services, handle, map->map_key); } - /* exit_boot_services() was called, thus cannot free */ - if (status != EFI_SUCCESS) - goto fail; - - return EFI_SUCCESS; - -free_map: - efi_bs_call(free_pool, *map->map); -fail: return status; } @@ -560,20 +550,16 @@ static const struct { * * %EFI_SUCCESS if the initrd was loaded successfully, in which * case @load_addr and @load_size are assigned accordingly * * %EFI_NOT_FOUND if no LoadFile2 protocol exists on the initrd device path - * * %EFI_INVALID_PARAMETER if load_addr == NULL or load_size == NULL * * %EFI_OUT_OF_RESOURCES if memory allocation failed * * %EFI_LOAD_ERROR in all other cases */ static -efi_status_t efi_load_initrd_dev_path(unsigned long *load_addr, - unsigned long *load_size, +efi_status_t efi_load_initrd_dev_path(struct linux_efi_initrd *initrd, unsigned long max) { efi_guid_t lf2_proto_guid = EFI_LOAD_FILE2_PROTOCOL_GUID; efi_device_path_protocol_t *dp; efi_load_file2_protocol_t *lf2; - unsigned long initrd_addr; - unsigned long initrd_size; efi_handle_t handle; efi_status_t status; @@ -587,75 +573,94 @@ efi_status_t efi_load_initrd_dev_path(unsigned long *load_addr, if (status != EFI_SUCCESS) return status; - status = efi_call_proto(lf2, load_file, dp, false, &initrd_size, NULL); + initrd->size = 0; + status = efi_call_proto(lf2, load_file, dp, false, &initrd->size, NULL); if (status != EFI_BUFFER_TOO_SMALL) return EFI_LOAD_ERROR; - status = efi_allocate_pages(initrd_size, &initrd_addr, max); + status = efi_allocate_pages(initrd->size, &initrd->base, max); if (status != EFI_SUCCESS) return status; - status = efi_call_proto(lf2, load_file, dp, false, &initrd_size, - (void *)initrd_addr); + status = efi_call_proto(lf2, load_file, dp, false, &initrd->size, + (void *)initrd->base); if (status != EFI_SUCCESS) { - efi_free(initrd_size, initrd_addr); + efi_free(initrd->size, initrd->base); return EFI_LOAD_ERROR; } - - *load_addr = initrd_addr; - *load_size = initrd_size; return EFI_SUCCESS; } static efi_status_t efi_load_initrd_cmdline(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, + struct linux_efi_initrd *initrd, unsigned long soft_limit, unsigned long hard_limit) { if (!IS_ENABLED(CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER) || - (IS_ENABLED(CONFIG_X86) && (!efi_is_native() || image == NULL))) { - *load_addr = *load_size = 0; - return EFI_SUCCESS; - } + (IS_ENABLED(CONFIG_X86) && (!efi_is_native() || image == NULL))) + return EFI_UNSUPPORTED; return handle_cmdline_files(image, L"initrd=", sizeof(L"initrd=") - 2, soft_limit, hard_limit, - load_addr, load_size); + &initrd->base, &initrd->size); } /** * efi_load_initrd() - Load initial RAM disk * @image: EFI loaded image protocol - * @load_addr: pointer to loaded initrd - * @load_size: size of loaded initrd * @soft_limit: preferred size of allocated memory for loading the initrd * @hard_limit: minimum size of allocated memory * * Return: status code */ efi_status_t efi_load_initrd(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, unsigned long soft_limit, - unsigned long hard_limit) + unsigned long hard_limit, + const struct linux_efi_initrd **out) { - efi_status_t status; + efi_guid_t tbl_guid = LINUX_EFI_INITRD_MEDIA_GUID; + efi_status_t status = EFI_SUCCESS; + struct linux_efi_initrd initrd, *tbl; - if (!load_addr || !load_size) - return EFI_INVALID_PARAMETER; + if (!IS_ENABLED(CONFIG_BLK_DEV_INITRD) || efi_noinitrd) + return EFI_SUCCESS; - status = efi_load_initrd_dev_path(load_addr, load_size, hard_limit); + status = efi_load_initrd_dev_path(&initrd, hard_limit); if (status == EFI_SUCCESS) { efi_info("Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path\n"); } else if (status == EFI_NOT_FOUND) { - status = efi_load_initrd_cmdline(image, load_addr, load_size, - soft_limit, hard_limit); - if (status == EFI_SUCCESS && *load_size > 0) + status = efi_load_initrd_cmdline(image, &initrd, soft_limit, + hard_limit); + /* command line loader disabled or no initrd= passed? */ + if (status == EFI_UNSUPPORTED || status == EFI_NOT_READY) + return EFI_SUCCESS; + if (status == EFI_SUCCESS) efi_info("Loaded initrd from command line option\n"); } + if (status != EFI_SUCCESS) + goto failed; + + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, sizeof(initrd), + (void **)&tbl); + if (status != EFI_SUCCESS) + goto free_initrd; + + *tbl = initrd; + status = efi_bs_call(install_configuration_table, &tbl_guid, tbl); + if (status != EFI_SUCCESS) + goto free_tbl; + + if (out) + *out = tbl; + return EFI_SUCCESS; +free_tbl: + efi_bs_call(free_pool, tbl); +free_initrd: + efi_free(initrd.size, initrd.base); +failed: + efi_err("Failed to load initrd: 0x%lx\n", status); return status; } diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c index 0ab439c53eee3ea1a924cc98be844cd0ee19a3c3..fd95a6e97f2fda860bda889733294ee70d69b885 100644 --- a/drivers/firmware/efi/libstub/efi-stub.c +++ b/drivers/firmware/efi/libstub/efi-stub.c @@ -10,7 +10,6 @@ */ #include -#include #include #include "efistub.h" @@ -40,12 +39,18 @@ #ifdef CONFIG_ARM64 # define EFI_RT_VIRTUAL_LIMIT DEFAULT_MAP_WINDOW_64 +#elif defined(CONFIG_RISCV) || defined(CONFIG_LOONGARCH) +# define EFI_RT_VIRTUAL_LIMIT TASK_SIZE_MIN #else # define EFI_RT_VIRTUAL_LIMIT TASK_SIZE #endif +#ifndef EFI_RT_VIRTUAL_OFFSET +#define EFI_RT_VIRTUAL_OFFSET 0 +#endif + static u64 virtmap_base = EFI_RT_VIRTUAL_BASE; -static bool flat_va_mapping; +static bool flat_va_mapping = !!EFI_RT_VIRTUAL_OFFSET; const efi_system_table_t *efi_system_table; @@ -122,16 +127,11 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, unsigned long image_addr; unsigned long image_size = 0; /* addr/point and size pairs for memory management*/ - unsigned long initrd_addr = 0; - unsigned long initrd_size = 0; - unsigned long fdt_addr = 0; /* Original DTB */ - unsigned long fdt_size = 0; char *cmdline_ptr = NULL; int cmdline_size = 0; efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID; unsigned long reserve_addr = 0; unsigned long reserve_size = 0; - enum efi_secureboot_mode secure_boot; struct screen_info *si; efi_properties_table_t *prop_tbl; unsigned long max_addr; @@ -208,42 +208,10 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, /* Ask the firmware to clear memory on unclean shutdown */ efi_enable_reset_attack_mitigation(); - secure_boot = efi_get_secureboot(); - - /* - * Unauthenticated device tree data is a security hazard, so ignore - * 'dtb=' unless UEFI Secure Boot is disabled. We assume that secure - * boot is enabled if we can't determine its state. - */ - if (!IS_ENABLED(CONFIG_EFI_ARMSTUB_DTB_LOADER) || - secure_boot != efi_secureboot_mode_disabled) { - if (strstr(cmdline_ptr, "dtb=")) - efi_err("Ignoring DTB from command line.\n"); - } else { - status = efi_load_dtb(image, &fdt_addr, &fdt_size); - - if (status != EFI_SUCCESS) { - efi_err("Failed to load device tree!\n"); - goto fail_free_image; - } - } - - if (fdt_addr) { - efi_info("Using DTB from command line\n"); - } else { - /* Look for a device tree configuration table entry. */ - fdt_addr = (uintptr_t)get_fdt(&fdt_size); - if (fdt_addr) - efi_info("Using DTB from configuration table\n"); - } - - if (!fdt_addr) - efi_info("Generating empty DTB\n"); - if (!efi_noinitrd) { max_addr = efi_get_max_initrd_addr(image_addr); - status = efi_load_initrd(image, &initrd_addr, &initrd_size, - ULONG_MAX, max_addr); + efi_load_initrd(image, ULONG_MAX, efi_get_max_initrd_addr(image_addr), + NULL); if (status != EFI_SUCCESS) efi_err("Failed to load initrd!\n"); } @@ -258,8 +226,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, * The easiest way to achieve that is to simply use a 1:1 mapping. */ prop_tbl = get_efi_config_table(EFI_PROPERTIES_TABLE_GUID); - flat_va_mapping = prop_tbl && - (prop_tbl->memory_protection_attribute & + flat_va_mapping |= prop_tbl && + (prop_tbl->memory_protection_attribute & EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA); /* force efi_novamap if SetVirtualAddressMap() is unsupported */ @@ -288,26 +256,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, install_memreserve_table(); - status = allocate_new_fdt_and_exit_boot(handle, &fdt_addr, - efi_get_max_fdt_addr(image_addr), - initrd_addr, initrd_size, - cmdline_ptr, fdt_addr, fdt_size); - if (status != EFI_SUCCESS) - goto fail_free_initrd; - - if (IS_ENABLED(CONFIG_ARM)) - efi_handle_post_ebs_state(); - - efi_enter_kernel(image_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); - /* not reached */ + status = efi_boot_kernel(handle, image, image_addr, cmdline_ptr); -fail_free_initrd: - efi_err("Failed to update FDT and exit boot services\n"); - - efi_free(initrd_size, initrd_addr); - efi_free(fdt_size, fdt_addr); - -fail_free_image: efi_free(image_size, image_addr); efi_free(reserve_size, reserve_addr); fail_free_screeninfo: @@ -318,6 +268,35 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, return status; } +/* + * efi_allocate_virtmap() - create a pool allocation for the virtmap + * + * Create an allocation that is of sufficient size to hold all the memory + * descriptors that will be passed to SetVirtualAddressMap() to inform the + * firmware about the virtual mapping that will be used under the OS to call + * into the firmware. + */ +efi_status_t efi_alloc_virtmap(efi_memory_desc_t **virtmap, + unsigned long *desc_size, u32 *desc_ver) +{ + unsigned long size, mmap_key; + efi_status_t status; + + /* + * Use the size of the current memory map as an upper bound for the + * size of the buffer we need to pass to SetVirtualAddressMap() to + * cover all EFI_MEMORY_RUNTIME regions. + */ + size = 0; + status = efi_bs_call(get_memory_map, &size, NULL, &mmap_key, desc_size, + desc_ver); + if (status != EFI_BUFFER_TOO_SMALL) + return EFI_LOAD_ERROR; + + return efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, + (void **)virtmap); +} + /* * efi_get_virtmap() - create a virtual mapping for the EFI memory map * @@ -333,6 +312,8 @@ void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, efi_memory_desc_t *in, *out = runtime_map; int l; + *count = 0; + for (l = 0; l < map_size; l += desc_size) { u64 paddr, size; @@ -343,7 +324,7 @@ void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, paddr = in->phys_addr; size = in->num_pages * EFI_PAGE_SIZE; - in->virt_addr = in->phys_addr; + in->virt_addr = in->phys_addr + EFI_RT_VIRTUAL_OFFSET; if (efi_novamap) { continue; } diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index c5e0c6e99d20c200cdab15cf28b7372f916d87a9..881a698f96a7140b05335428ca24adb82be6ad9e 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -157,15 +157,6 @@ void efi_set_u64_split(u64 data, u32 *lo, u32 *hi) */ #define EFI_MMAP_NR_SLACK_SLOTS 8 -struct efi_boot_memmap { - efi_memory_desc_t **map; - unsigned long *map_size; - unsigned long *desc_size; - u32 *desc_ver; - unsigned long *key_ptr; - unsigned long *buff_size; -}; - typedef struct efi_generic_dev_path efi_device_path_protocol_t; typedef void *efi_event_t; @@ -743,21 +734,16 @@ typedef efi_status_t (*efi_exit_boot_map_processing)( struct efi_boot_memmap *map, void *priv); -efi_status_t efi_exit_boot_services(void *handle, - struct efi_boot_memmap *map, - void *priv, +efi_status_t efi_exit_boot_services(void *handle, void *priv, efi_exit_boot_map_processing priv_func); -efi_status_t allocate_new_fdt_and_exit_boot(void *handle, - unsigned long *new_fdt_addr, - unsigned long max_addr, - u64 initrd_addr, u64 initrd_size, - char *cmdline_ptr, - unsigned long fdt_addr, - unsigned long fdt_size); +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr); void *get_fdt(unsigned long *fdt_size); +efi_status_t efi_alloc_virtmap(efi_memory_desc_t **virtmap, + unsigned long *desc_size, u32 *desc_ver); void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, unsigned long desc_size, efi_memory_desc_t *runtime_map, int *count); @@ -785,7 +771,8 @@ void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_si char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len); -efi_status_t efi_get_memory_map(struct efi_boot_memmap *map); +efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, + bool install_cfg_tbl); efi_status_t efi_allocate_pages(unsigned long size, unsigned long *addr, unsigned long max); @@ -828,10 +815,9 @@ static inline efi_status_t efi_load_dtb(efi_loaded_image_t *image, } efi_status_t efi_load_initrd(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, unsigned long soft_limit, - unsigned long hard_limit); + unsigned long hard_limit, + const struct linux_efi_initrd **out); /* * This function handles the architcture specific differences between arm and * arm64 regarding where the kernel image must be loaded and any memory that diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index d48b0de05b62fdab81b9b0a7033766a1ceac0e75..6a337f1f8787b3bc68f771500f9fcd4b44a7b4f7 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -28,8 +28,7 @@ static void fdt_update_cell_size(void *fdt) } static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size, - void *fdt, int new_fdt_size, char *cmdline_ptr, - u64 initrd_addr, u64 initrd_size) + void *fdt, int new_fdt_size, char *cmdline_ptr) { int node, num_rsv; int status; @@ -93,21 +92,6 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size, goto fdt_set_fail; } - /* Set initrd address/end in device tree, if present */ - if (initrd_size != 0) { - u64 initrd_image_end; - u64 initrd_image_start = cpu_to_fdt64(initrd_addr); - - status = fdt_setprop_var(fdt, node, "linux,initrd-start", initrd_image_start); - if (status) - goto fdt_set_fail; - - initrd_image_end = cpu_to_fdt64(initrd_addr + initrd_size); - status = fdt_setprop_var(fdt, node, "linux,initrd-end", initrd_image_end); - if (status) - goto fdt_set_fail; - } - /* Add FDT entries for EFI runtime services in chosen node. */ node = fdt_subnode_offset(fdt, 0, "chosen"); fdt_val64 = cpu_to_fdt64((u64)(unsigned long)efi_system_table); @@ -170,25 +154,25 @@ static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map) if (node < 0) return EFI_LOAD_ERROR; - fdt_val64 = cpu_to_fdt64((unsigned long)*map->map); + fdt_val64 = cpu_to_fdt64((unsigned long)map->map); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-start", fdt_val64); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->map_size); + fdt_val32 = cpu_to_fdt32(map->map_size); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-size", fdt_val32); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->desc_size); + fdt_val32 = cpu_to_fdt32(map->desc_size); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-desc-size", fdt_val32); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->desc_ver); + fdt_val32 = cpu_to_fdt32(map->desc_ver); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-desc-ver", fdt_val32); if (err) @@ -198,22 +182,25 @@ static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map) } struct exit_boot_struct { + struct efi_boot_memmap *boot_memmap; efi_memory_desc_t *runtime_map; - int *runtime_entry_count; + int runtime_entry_count; void *new_fdt_addr; }; -static efi_status_t exit_boot_func(struct efi_boot_memmap *map, - void *priv) +static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) { struct exit_boot_struct *p = priv; + + p->boot_memmap = map; + /* * Update the memory map with virtual addresses. The function will also * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME * entries so that we can pass it straight to SetVirtualAddressMap() */ - efi_get_virtmap(*map->map, *map->map_size, *map->desc_size, - p->runtime_map, p->runtime_entry_count); + efi_get_virtmap(map->map, map->map_size, map->desc_size, + p->runtime_map, &p->runtime_entry_count); return update_fdt_memmap(p->new_fdt_addr, map); } @@ -223,79 +210,86 @@ static efi_status_t exit_boot_func(struct efi_boot_memmap *map, #endif /* - * Allocate memory for a new FDT, then add EFI, commandline, and - * initrd related fields to the FDT. This routine increases the - * FDT allocation size until the allocated memory is large - * enough. EFI allocations are in EFI_PAGE_SIZE granules, - * which are fixed at 4K bytes, so in most cases the first - * allocation should succeed. - * EFI boot services are exited at the end of this function. - * There must be no allocations between the get_memory_map() - * call and the exit_boot_services() call, so the exiting of - * boot services is very tightly tied to the creation of the FDT - * with the final memory map in it. + * Allocate memory for a new FDT, then add EFI and commandline related fields + * to the FDT. This routine increases the FDT allocation size until the + * allocated memory is large enough. EFI allocations are in EFI_PAGE_SIZE + * granules, which are fixed at 4K bytes, so in most cases the first allocation + * should succeed. EFI boot services are exited at the end of this function. + * There must be no allocations between the get_memory_map() call and the + * exit_boot_services() call, so the exiting of boot services is very tightly + * tied to the creation of the FDT with the final memory map in it. */ - +static efi_status_t allocate_new_fdt_and_exit_boot(void *handle, + efi_loaded_image_t *image, unsigned long *new_fdt_addr, - unsigned long max_addr, - u64 initrd_addr, u64 initrd_size, - char *cmdline_ptr, - unsigned long fdt_addr, - unsigned long fdt_size) + char *cmdline_ptr) { - unsigned long map_size, desc_size, buff_size; + unsigned long desc_size; u32 desc_ver; - unsigned long mmap_key; - efi_memory_desc_t *memory_map, *runtime_map; efi_status_t status; - int runtime_entry_count; - struct efi_boot_memmap map; struct exit_boot_struct priv; + unsigned long fdt_addr = 0; + unsigned long fdt_size = 0; - map.map = &runtime_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = &desc_ver; - map.key_ptr = &mmap_key; - map.buff_size = &buff_size; + if (!efi_novamap) { + status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, + &desc_ver); + if (status != EFI_SUCCESS) { + efi_err("Unable to retrieve UEFI memory map.\n"); + return status; + } + } /* - * Get a copy of the current memory map that we will use to prepare - * the input for SetVirtualAddressMap(). We don't have to worry about - * subsequent allocations adding entries, since they could not affect - * the number of EFI_MEMORY_RUNTIME regions. + * Unauthenticated device tree data is a security hazard, so ignore + * 'dtb=' unless UEFI Secure Boot is disabled. We assume that secure + * boot is enabled if we can't determine its state. */ - status = efi_get_memory_map(&map); - if (status != EFI_SUCCESS) { - efi_err("Unable to retrieve UEFI memory map.\n"); - return status; + if (!IS_ENABLED(CONFIG_EFI_ARMSTUB_DTB_LOADER) || + efi_get_secureboot() != efi_secureboot_mode_disabled) { + if (strstr(cmdline_ptr, "dtb=")) + efi_err("Ignoring DTB from command line.\n"); + } else { + status = efi_load_dtb(image, &fdt_addr, &fdt_size); + + if (status != EFI_SUCCESS && status != EFI_NOT_READY) { + efi_err("Failed to load device tree!\n"); + goto fail; + } + } + + if (fdt_addr) { + efi_info("Using DTB from command line\n"); + } else { + /* Look for a device tree configuration table entry. */ + fdt_addr = (uintptr_t)get_fdt(&fdt_size); + if (fdt_addr) + efi_info("Using DTB from configuration table\n"); } - efi_info("Exiting boot services and installing virtual address map...\n"); + if (!fdt_addr) + efi_info("Generating empty DTB\n"); + + efi_info("Exiting boot services...\n"); - map.map = &memory_map; - status = efi_allocate_pages(MAX_FDT_SIZE, new_fdt_addr, max_addr); + status = efi_allocate_pages(MAX_FDT_SIZE, new_fdt_addr, ULONG_MAX); if (status != EFI_SUCCESS) { efi_err("Unable to allocate memory for new device tree.\n"); goto fail; } status = update_fdt((void *)fdt_addr, fdt_size, - (void *)*new_fdt_addr, MAX_FDT_SIZE, cmdline_ptr, - initrd_addr, initrd_size); + (void *)*new_fdt_addr, MAX_FDT_SIZE, cmdline_ptr); if (status != EFI_SUCCESS) { efi_err("Unable to construct new device tree.\n"); goto fail_free_new_fdt; } - runtime_entry_count = 0; - priv.runtime_map = runtime_map; - priv.runtime_entry_count = &runtime_entry_count; - priv.new_fdt_addr = (void *)*new_fdt_addr; + priv.new_fdt_addr = (void *)*new_fdt_addr; - status = efi_exit_boot_services(handle, &map, &priv, exit_boot_func); + status = efi_exit_boot_services(handle, &priv, exit_boot_func); if (status == EFI_SUCCESS) { efi_set_virtual_address_map_t *svam; @@ -305,8 +299,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, /* Install the new virtual address map */ svam = efi_system_table->runtime->set_virtual_address_map; - status = svam(runtime_entry_count * desc_size, desc_size, - desc_ver, runtime_map); + status = svam(priv.runtime_entry_count * desc_size, desc_size, + desc_ver, priv.runtime_map); /* * We are beyond the point of no return here, so if the call to @@ -314,19 +308,21 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, * incoming kernel but proceed normally otherwise. */ if (status != EFI_SUCCESS) { + efi_memory_desc_t *p; int l; /* * Set the virtual address field of all - * EFI_MEMORY_RUNTIME entries to 0. This will signal - * the incoming kernel that no virtual translation has - * been installed. + * EFI_MEMORY_RUNTIME entries to U64_MAX. This will + * signal the incoming kernel that no virtual + * translation has been installed. */ - for (l = 0; l < map_size; l += desc_size) { - efi_memory_desc_t *p = (void *)memory_map + l; + for (l = 0; l < priv.boot_memmap->map_size; + l += priv.boot_memmap->desc_size) { + p = (void *)priv.boot_memmap->map + l; if (p->attribute & EFI_MEMORY_RUNTIME) - p->virt_addr = 0; + p->virt_addr = U64_MAX; } } return EFI_SUCCESS; @@ -338,11 +334,33 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, efi_free(MAX_FDT_SIZE, *new_fdt_addr); fail: - efi_system_table->boottime->free_pool(runtime_map); + efi_free(fdt_size, fdt_addr); + if (!efi_novamap) + efi_bs_call(free_pool, priv.runtime_map); return EFI_LOAD_ERROR; } +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr) +{ + unsigned long fdt_addr; + efi_status_t status; + + status = allocate_new_fdt_and_exit_boot(handle, image, &fdt_addr, + cmdline_ptr); + if (status != EFI_SUCCESS) { + efi_err("Failed to update FDT and exit boot services\n"); + return status; + } + + if (IS_ENABLED(CONFIG_ARM)) + efi_handle_post_ebs_state(); + + efi_enter_kernel(kernel_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); + /* not reached */ +} + void *get_fdt(unsigned long *fdt_size) { void *fdt; diff --git a/drivers/firmware/efi/libstub/file.c b/drivers/firmware/efi/libstub/file.c index dd95f330fe6e173ef8bed69f2b59b7e2cbaddb57..488a8027518d27dc3911b62ac3f06a1c919653d5 100644 --- a/drivers/firmware/efi/libstub/file.c +++ b/drivers/firmware/efi/libstub/file.c @@ -238,6 +238,9 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, if (volume) volume->close(volume); + + if (*load_size == 0) + return EFI_NOT_READY; return EFI_SUCCESS; err_close_file: diff --git a/drivers/firmware/efi/libstub/loongarch-stub.c b/drivers/firmware/efi/libstub/loongarch-stub.c new file mode 100644 index 0000000000000000000000000000000000000000..1aefb84d1fc8fdf2034cea9afd9d76f0bd2d3dcd --- /dev/null +++ b/drivers/firmware/efi/libstub/loongarch-stub.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Yun Liu + * Huacai Chen + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include "efistub.h" + +#define BOOT_HEAP_SIZE 0x400000 + +typedef void __noreturn (*kernel_entry_t)(bool efi, unsigned long cmdline, + unsigned long systab, long kdump_reloc_offset); + +extern long kernel_entaddr; +extern void decompress_kernel(unsigned long boot_heap_start, long kdump_reloc_offset); + +static unsigned char efi_heap[BOOT_HEAP_SIZE]; +static efi_guid_t screen_info_guid = LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID; +static kernel_entry_t kernel_entry; + +struct screen_info *alloc_screen_info(void) +{ + efi_status_t status; + struct screen_info *si; + + status = efi_bs_call(allocate_pool, + EFI_RUNTIME_SERVICES_DATA, sizeof(*si), (void **)&si); + if (status != EFI_SUCCESS) + return NULL; + + memset(si, 0, sizeof(*si)); + + status = efi_bs_call(install_configuration_table, &screen_info_guid, si); + if (status == EFI_SUCCESS) + return si; + + efi_bs_call(free_pool, si); + + return NULL; +} + +void free_screen_info(struct screen_info *si) +{ + if (!si) + return; + + efi_bs_call(install_configuration_table, &screen_info_guid, NULL); + efi_bs_call(free_pool, si); +} + +efi_status_t check_platform_features(void) +{ + return EFI_SUCCESS; +} + +efi_status_t handle_kernel_image(unsigned long *image_addr, + unsigned long *image_size, + unsigned long *reserve_addr, + unsigned long *reserve_size, + efi_loaded_image_t *image) +{ + /* Config Direct Mapping */ + csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); + csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); + + decompress_kernel((unsigned long)efi_heap, 0); + kernel_entry = (kernel_entry_t)kernel_entaddr; + + return EFI_SUCCESS; +} + +struct exit_boot_struct { + efi_memory_desc_t *runtime_map; + int runtime_entry_count; +}; + +static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) +{ + struct exit_boot_struct *p = priv; + + /* + * Update the memory map with virtual addresses. The function will also + * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME + * entries so that we can pass it straight to SetVirtualAddressMap() + */ + efi_get_virtmap(map->map, map->map_size, map->desc_size, + p->runtime_map, &p->runtime_entry_count); + + return EFI_SUCCESS; +} + +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr) +{ + struct exit_boot_struct priv; + unsigned long desc_size; + efi_status_t status; + u32 desc_ver; + + status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, &desc_ver); + if (status != EFI_SUCCESS) { + efi_err("Unable to retrieve UEFI memory map.\n"); + return status; + } + + efi_info("Exiting boot services\n"); + + efi_novamap = false; + status = efi_exit_boot_services(handle, &priv, exit_boot_func); + if (status != EFI_SUCCESS) + return status; + + /* Install the new virtual address map */ + efi_rt_call(set_virtual_address_map, + priv.runtime_entry_count * desc_size, desc_size, + desc_ver, priv.runtime_map); + + kernel_entry(true, (unsigned long)cmdline_ptr, (unsigned long)efi_system_table, 0); +} diff --git a/drivers/firmware/efi/libstub/mem.c b/drivers/firmware/efi/libstub/mem.c index feef8d4be113b653d8216347ed81e7beaf55396e..45841ef55a9f6ba284ec7f4bd34fc24a6b91451e 100644 --- a/drivers/firmware/efi/libstub/mem.c +++ b/drivers/firmware/efi/libstub/mem.c @@ -5,71 +5,66 @@ #include "efistub.h" -static inline bool mmap_has_headroom(unsigned long buff_size, - unsigned long map_size, - unsigned long desc_size) -{ - unsigned long slack = buff_size - map_size; - - return slack / desc_size >= EFI_MMAP_NR_SLACK_SLOTS; -} - /** * efi_get_memory_map() - get memory map - * @map: on return pointer to memory map + * @map: pointer to memory map pointer to which to assign the + * newly allocated memory map + * @install_cfg_tbl: whether or not to install the boot memory map as a + * configuration table * * Retrieve the UEFI memory map. The allocated memory leaves room for * up to EFI_MMAP_NR_SLACK_SLOTS additional memory map entries. * * Return: status code */ -efi_status_t efi_get_memory_map(struct efi_boot_memmap *map) +efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, + bool install_cfg_tbl) { - efi_memory_desc_t *m = NULL; + int memtype = install_cfg_tbl ? EFI_ACPI_RECLAIM_MEMORY + : EFI_LOADER_DATA; + efi_guid_t tbl_guid = LINUX_EFI_BOOT_MEMMAP_GUID; + struct efi_boot_memmap *m, tmp; efi_status_t status; - unsigned long key; - u32 desc_version; - - *map->desc_size = sizeof(*m); - *map->map_size = *map->desc_size * 32; - *map->buff_size = *map->map_size; -again: - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, - *map->map_size, (void **)&m); + unsigned long size; + + tmp.map_size = 0; + status = efi_bs_call(get_memory_map, &tmp.map_size, NULL, &tmp.map_key, + &tmp.desc_size, &tmp.desc_ver); + if (status != EFI_BUFFER_TOO_SMALL) + return EFI_LOAD_ERROR; + + size = tmp.map_size + tmp.desc_size * EFI_MMAP_NR_SLACK_SLOTS; + status = efi_bs_call(allocate_pool, memtype, sizeof(*m) + size, + (void **)&m); if (status != EFI_SUCCESS) - goto fail; - - *map->desc_size = 0; - key = 0; - status = efi_bs_call(get_memory_map, map->map_size, m, - &key, map->desc_size, &desc_version); - if (status == EFI_BUFFER_TOO_SMALL || - !mmap_has_headroom(*map->buff_size, *map->map_size, - *map->desc_size)) { - efi_bs_call(free_pool, m); + return status; + + if (install_cfg_tbl) { /* - * Make sure there is some entries of headroom so that the - * buffer can be reused for a new map after allocations are - * no longer permitted. Its unlikely that the map will grow to - * exceed this headroom once we are ready to trigger - * ExitBootServices() + * Installing a configuration table might allocate memory, and + * this may modify the memory map. This means we should install + * the configuration table first, and re-install or delete it + * as needed. */ - *map->map_size += *map->desc_size * EFI_MMAP_NR_SLACK_SLOTS; - *map->buff_size = *map->map_size; - goto again; + status = efi_bs_call(install_configuration_table, &tbl_guid, m); + if (status != EFI_SUCCESS) + goto free_map; } - if (status == EFI_SUCCESS) { - if (map->key_ptr) - *map->key_ptr = key; - if (map->desc_ver) - *map->desc_ver = desc_version; - } else { - efi_bs_call(free_pool, m); - } + m->buff_size = m->map_size = size; + status = efi_bs_call(get_memory_map, &m->map_size, m->map, &m->map_key, + &m->desc_size, &m->desc_ver); + if (status != EFI_SUCCESS) + goto uninstall_table; + + *map = m; + return EFI_SUCCESS; -fail: - *map->map = m; +uninstall_table: + if (install_cfg_tbl) + efi_bs_call(install_configuration_table, &tbl_guid, NULL); +free_map: + efi_bs_call(free_pool, m); return status; } diff --git a/drivers/firmware/efi/libstub/randomalloc.c b/drivers/firmware/efi/libstub/randomalloc.c index 724155b9e10dcf84a44a836efed036574daaa728..4f15e6103a2efc92efbc05ebe073fc4f611fe73e 100644 --- a/drivers/firmware/efi/libstub/randomalloc.c +++ b/drivers/firmware/efi/libstub/randomalloc.c @@ -55,21 +55,12 @@ efi_status_t efi_random_alloc(unsigned long size, unsigned long *addr, unsigned long random_seed) { - unsigned long map_size, desc_size, total_slots = 0, target_slot; - unsigned long buff_size; + unsigned long total_slots = 0, target_slot; + struct efi_boot_memmap *map; efi_status_t status; - efi_memory_desc_t *memory_map; int map_offset; - struct efi_boot_memmap map; - map.map = &memory_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = NULL; - map.key_ptr = NULL; - map.buff_size = &buff_size; - - status = efi_get_memory_map(&map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) return status; @@ -79,8 +70,8 @@ efi_status_t efi_random_alloc(unsigned long size, size = round_up(size, EFI_ALLOC_ALIGN); /* count the suitable slots in each memory map entry */ - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; unsigned long slots; slots = get_entry_num_slots(md, size, ilog2(align)); @@ -102,8 +93,8 @@ efi_status_t efi_random_alloc(unsigned long size, * to calculate the randomly chosen address, and allocate it directly * using EFI_ALLOCATE_ADDRESS. */ - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; efi_physical_addr_t target; unsigned long pages; @@ -122,7 +113,7 @@ efi_status_t efi_random_alloc(unsigned long size, break; } - efi_bs_call(free_pool, memory_map); + efi_bs_call(free_pool, map); return status; } diff --git a/drivers/firmware/efi/libstub/relocate.c b/drivers/firmware/efi/libstub/relocate.c index 8ee9eb2b90392d993431bba9972923cc4c293683..bf6fbd5d22a1a53af196be2a15f5593468f78da4 100644 --- a/drivers/firmware/efi/libstub/relocate.c +++ b/drivers/firmware/efi/libstub/relocate.c @@ -23,21 +23,12 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, unsigned long *addr, unsigned long min) { - unsigned long map_size, desc_size, buff_size; - efi_memory_desc_t *map; + struct efi_boot_memmap *map; efi_status_t status; unsigned long nr_pages; int i; - struct efi_boot_memmap boot_map; - boot_map.map = ↦ - boot_map.map_size = &map_size; - boot_map.desc_size = &desc_size; - boot_map.desc_ver = NULL; - boot_map.key_ptr = NULL; - boot_map.buff_size = &buff_size; - - status = efi_get_memory_map(&boot_map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) goto fail; @@ -52,12 +43,12 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, size = round_up(size, EFI_ALLOC_ALIGN); nr_pages = size / EFI_PAGE_SIZE; - for (i = 0; i < map_size / desc_size; i++) { + for (i = 0; i < map->map_size / map->desc_size; i++) { efi_memory_desc_t *desc; - unsigned long m = (unsigned long)map; + unsigned long m = (unsigned long)map->map; u64 start, end; - desc = efi_early_memdesc_ptr(m, desc_size, i); + desc = efi_early_memdesc_ptr(m, map->desc_size, i); if (desc->type != EFI_CONVENTIONAL_MEMORY) continue; @@ -87,7 +78,7 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, } } - if (i == map_size / desc_size) + if (i == map->map_size / map->desc_size) status = EFI_NOT_FOUND; efi_bs_call(free_pool, map); diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 9f998e6bff957e2e5adff406f8b1ed5c9f141237..ff960ed7bc7e2199dea96e50d03d4751c0da25f8 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -620,32 +620,22 @@ static efi_status_t exit_boot_func(struct efi_boot_memmap *map, efi_set_u64_split((unsigned long)efi_system_table, &p->efi->efi_systab, &p->efi->efi_systab_hi); - p->efi->efi_memdesc_size = *map->desc_size; - p->efi->efi_memdesc_version = *map->desc_ver; - efi_set_u64_split((unsigned long)*map->map, + p->efi->efi_memdesc_size = map->desc_size; + p->efi->efi_memdesc_version = map->desc_ver; + efi_set_u64_split((unsigned long)map->map, &p->efi->efi_memmap, &p->efi->efi_memmap_hi); - p->efi->efi_memmap_size = *map->map_size; + p->efi->efi_memmap_size = map->map_size; return EFI_SUCCESS; } static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) { - unsigned long map_sz, key, desc_size, buff_size; - efi_memory_desc_t *mem_map; struct setup_data *e820ext = NULL; __u32 e820ext_size = 0; efi_status_t status; - __u32 desc_version; - struct efi_boot_memmap map; struct exit_boot_struct priv; - map.map = &mem_map; - map.map_size = &map_sz; - map.desc_size = &desc_size; - map.desc_ver = &desc_version; - map.key_ptr = &key; - map.buff_size = &buff_size; priv.boot_params = boot_params; priv.efi = &boot_params->efi_info; @@ -654,7 +644,7 @@ static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) return status; /* Might as well exit boot services now */ - status = efi_exit_boot_services(handle, &map, &priv, exit_boot_func); + status = efi_exit_boot_services(handle, &priv, exit_boot_func); if (status != EFI_SUCCESS) return status; @@ -680,6 +670,7 @@ unsigned long efi_main(efi_handle_t handle, unsigned long bzimage_addr = (unsigned long)startup_32; unsigned long buffer_start, buffer_end; struct setup_header *hdr = &boot_params->hdr; + const struct linux_efi_initrd *initrd = NULL; efi_status_t status; efi_system_table = sys_table_arg; @@ -768,17 +759,17 @@ unsigned long efi_main(efi_handle_t handle, if (!efi_noinitrd) { unsigned long addr, size; - status = efi_load_initrd(image, &addr, &size, - hdr->initrd_addr_max, ULONG_MAX); + status = efi_load_initrd(image, hdr->initrd_addr_max, ULONG_MAX, + &initrd); if (status != EFI_SUCCESS) { efi_err("Failed to load initrd!\n"); goto fail; } - if (size > 0) { - efi_set_u64_split(addr, &hdr->ramdisk_image, + if (initrd && initrd->size > 0) { + efi_set_u64_split(initrd->base, &hdr->ramdisk_image, &boot_params->ext_ramdisk_image); - efi_set_u64_split(size, &hdr->ramdisk_size, + efi_set_u64_split(initrd->size, &hdr->ramdisk_size, &boot_params->ext_ramdisk_size); } } diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 39f3e13664099dfcd72b6c3468d257a0aaaa373c..9e4059836d5d6dabb692c73bc7b85a81b5587846 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -350,7 +350,7 @@ config GPIO_LOGICVC config GPIO_LOONGSON bool "Loongson-2/3 GPIO support" - depends on CPU_LOONGSON2EF || CPU_LOONGSON64 + depends on CPU_LOONGSON2EF || CPU_LOONGSON64 || LOONGARCH help driver for GPIO functionality on Loongson-2F/3A/3B processors. diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c index 865ab2b34fdda6ba4d443498e2bd3bfdfd5ad1f5..3dfb8b6c6c710ae7eb9b572bc9748a5b2483d576 100644 --- a/drivers/gpio/gpio-eic-sprd.c +++ b/drivers/gpio/gpio-eic-sprd.c @@ -318,20 +318,27 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type) switch (flow_type) { case IRQ_TYPE_LEVEL_HIGH: sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 1); + sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IC, 1); break; case IRQ_TYPE_LEVEL_LOW: sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 0); + sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IC, 1); break; case IRQ_TYPE_EDGE_RISING: case IRQ_TYPE_EDGE_FALLING: case IRQ_TYPE_EDGE_BOTH: state = sprd_eic_get(chip, offset); - if (state) + if (state) { sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 0); - else + sprd_eic_update(chip, offset, + SPRD_EIC_DBNC_IC, 1); + } else { sprd_eic_update(chip, offset, SPRD_EIC_DBNC_IEV, 1); + sprd_eic_update(chip, offset, + SPRD_EIC_DBNC_IC, 1); + } break; default: return -ENOTSUPP; @@ -343,20 +350,27 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type) switch (flow_type) { case IRQ_TYPE_LEVEL_HIGH: sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 0); + sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTCLR, 1); break; case IRQ_TYPE_LEVEL_LOW: sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 1); + sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTCLR, 1); break; case IRQ_TYPE_EDGE_RISING: case IRQ_TYPE_EDGE_FALLING: case IRQ_TYPE_EDGE_BOTH: state = sprd_eic_get(chip, offset); - if (state) + if (state) { sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 0); - else + sprd_eic_update(chip, offset, + SPRD_EIC_LATCH_INTCLR, 1); + } else { sprd_eic_update(chip, offset, SPRD_EIC_LATCH_INTPOL, 1); + sprd_eic_update(chip, offset, + SPRD_EIC_LATCH_INTCLR, 1); + } break; default: return -ENOTSUPP; @@ -370,29 +384,34 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type) sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTPOL, 1); + sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_EDGE_FALLING: sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTPOL, 0); + sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_EDGE_BOTH: sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTBOTH, 1); + sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_LEVEL_HIGH: sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTMODE, 1); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTPOL, 1); + sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTCLR, 1); irq_set_handler_locked(data, handle_level_irq); break; case IRQ_TYPE_LEVEL_LOW: sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTMODE, 1); sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTPOL, 0); + sprd_eic_update(chip, offset, SPRD_EIC_ASYNC_INTCLR, 1); irq_set_handler_locked(data, handle_level_irq); break; default: @@ -405,29 +424,34 @@ static int sprd_eic_irq_set_type(struct irq_data *data, unsigned int flow_type) sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTPOL, 1); + sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_EDGE_FALLING: sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTPOL, 0); + sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_EDGE_BOTH: sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTMODE, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTBOTH, 1); + sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTCLR, 1); irq_set_handler_locked(data, handle_edge_irq); break; case IRQ_TYPE_LEVEL_HIGH: sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTMODE, 1); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTPOL, 1); + sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTCLR, 1); irq_set_handler_locked(data, handle_level_irq); break; case IRQ_TYPE_LEVEL_LOW: sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTBOTH, 0); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTMODE, 1); sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTPOL, 0); + sprd_eic_update(chip, offset, SPRD_EIC_SYNC_INTCLR, 1); irq_set_handler_locked(data, handle_level_irq); break; default: diff --git a/drivers/gpio/gpio-loongson.c b/drivers/gpio/gpio-loongson.c index a42145873cc92ab1628ed11b354cb648f5a09150..c47bcd33f1eb2d80a8232f58ba3db8532cbb60a1 100644 --- a/drivers/gpio/gpio-loongson.c +++ b/drivers/gpio/gpio-loongson.c @@ -22,10 +22,10 @@ #define STLS2F_N_GPIO 4 #define STLS3A_N_GPIO 16 -#ifdef CONFIG_CPU_LOONGSON64 -#define LOONGSON_N_GPIO STLS3A_N_GPIO -#else +#ifdef CONFIG_CPU_LOONGSON2EF #define LOONGSON_N_GPIO STLS2F_N_GPIO +#else +#define LOONGSON_N_GPIO STLS3A_N_GPIO #endif /* diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 44ee319da1b357c9cd105419c5593683373d40d1..12012e1645d7b878599a257f3f4d2731343581fe 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -1479,6 +1479,20 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = { .ignore_wake = "INT33FF:01@0", }, }, + { + /* + * Spurious wakeups from TP_ATTN# pin + * Found in BIOS 0.35 + * https://gitlab.freedesktop.org/drm/amd/-/issues/3073 + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "GPD"), + DMI_MATCH(DMI_PRODUCT_NAME, "G1619-04"), + }, + .driver_data = &(struct acpi_gpiolib_dmi_quirk) { + .ignore_wake = "PNP0C50:00@8", + }, + }, {} /* Terminating entry */ }; diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 4e9b3a95fa7c7e6d88d9c367c30d6a2c7348c131..b37e6660dd4e344f26e99717a9af4960e766dc4f 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -313,6 +313,8 @@ source "drivers/gpu/drm/udl/Kconfig" source "drivers/gpu/drm/ast/Kconfig" +source "drivers/gpu/drm/loongson/Kconfig" + source "drivers/gpu/drm/mgag200/Kconfig" source "drivers/gpu/drm/armada/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 81569009f884817ef0ff73e9c3d3dd58cdbd25a4..e9dd6847c9fa3ff9dbca5affa5dcaef6b587c442 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -86,6 +86,7 @@ obj-$(CONFIG_DRM_ROCKCHIP) +=rockchip/ obj-$(CONFIG_DRM_GMA500) += gma500/ obj-$(CONFIG_DRM_UDL) += udl/ obj-$(CONFIG_DRM_AST) += ast/ +obj-$(CONFIG_DRM_LOONGSON) +=loongson/ obj-$(CONFIG_DRM_ARMADA) += armada/ obj-$(CONFIG_DRM_ATMEL_HLCDC) += atmel-hlcdc/ obj-y += rcar-du/ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 3e573077368b398a6fb3092df8dfb436daf68089..8a1cb1de2b13aee97a8fd6f0e07e9b70dec08377 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -582,7 +582,7 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, int r; if (!adev->smc_rreg) - return -EPERM; + return -EOPNOTSUPP; if (size & 0x3 || *pos & 0x3) return -EINVAL; @@ -644,7 +644,7 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user * int r; if (!adev->smc_wreg) - return -EPERM; + return -EOPNOTSUPP; if (size & 0x3 || *pos & 0x3) return -EINVAL; diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 6c4cba09d23b699b884ddbb2381b5876f4f9a183..40279455beb4b46d059dab31597608dac6c8f46e 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -26,7 +26,7 @@ config DRM_AMD_DC_DCN config DRM_AMD_DC_DCN3_0 bool "DCN 3.0 family" - depends on DRM_AMD_DC && X86 + depends on DRM_AMD_DC && (X86 || LOONGARCH) depends on DRM_AMD_DC_DCN help Choose this option if you want to have diff --git a/drivers/gpu/drm/amd/display/dc/calcs/Makefile b/drivers/gpu/drm/amd/display/dc/calcs/Makefile index 4674aca8f20697f2550e91e2d29b06f128b4a7e1..b41b92d806b5d93b701d67662e0fcf13f34df3ab 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/Makefile +++ b/drivers/gpu/drm/amd/display/dc/calcs/Makefile @@ -33,6 +33,10 @@ ifdef CONFIG_PPC64 calcs_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_LOONGARCH +calcs_rcflags := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -53,6 +57,9 @@ endif CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calcs.o := $(calcs_ccflags) CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calc_auto.o := $(calcs_ccflags) CFLAGS_$(AMDDALPATH)/dc/calcs/dcn_calc_math.o := $(calcs_ccflags) -Wno-tautological-compare +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calcs.o := $(calcs_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calc_auto.o := $(calcs_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/calcs/dcn_calc_math.o := $(calcs_rcflags) BW_CALCS = dce_calcs.o bw_fixed.o custom_float.o diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile index 52b1ce775a1e8f7d0831d903496665291a7b9eb6..88a13880aecbadc06e14e07bd8e97586f0daad16 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile @@ -104,6 +104,10 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/clk_mgr/dcn21/rn_clk_mgr.o := $(call cc-option,-mno-gnu-attribute) endif +ifdef CONFIG_LOONGARCH +CFLAGS_REMOVE_$(AMDDALPATH)/dc/clk_mgr/dcn21/rn_clk_mgr.o := -msoft-float +endif + AMD_DAL_CLK_MGR_DCN21 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn21/,$(CLK_MGR_DCN21)) AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN21) @@ -114,6 +118,23 @@ ifdef CONFIG_DRM_AMD_DC_DCN3_0 ############################################################################### CLK_MGR_DCN30 = dcn30_clk_mgr.o dcn30_clk_mgr_smu_msg.o +# prevent build errors regarding soft-float vs hard-float FP ABI tags +# this code is currently unused on ppc64, as it applies to VanGogh APUs only +ifdef CONFIG_PPC64 +CFLAGS_$(AMDDALPATH)/dc/clk_mgr/dcn30/dcn30_clk_mgr.o := $(call cc-option,-mno-gnu-attribute) +endif + +# prevent build errors: +# ...: '-mgeneral-regs-only' is incompatible with the use of floating-point types +# this file is unused on arm64, just like on ppc64 +ifdef CONFIG_ARM64 +CFLAGS_REMOVE_$(AMDDALPATH)/dc/clk_mgr/dcn30/dcn30_clk_mgr.o := -mgeneral-regs-only +endif + +ifdef CONFIG_LOONGARCH +CFLAGS_REMOVE_$(AMDDALPATH)/dc/clk_mgr/dcn30/dcn30_clk_mgr.o := -msoft-float +endif + AMD_DAL_CLK_MGR_DCN30 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn30/,$(CLK_MGR_DCN30)) AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN30) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c index cab47bb211728ce0885258d0f42088060ef07c85..5e7cfd87bb72d73482fe9059adf55be982379c0b 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c @@ -104,7 +104,7 @@ static void dcn3_init_single_clock(struct clk_mgr_internal *clk_mgr, PPCLK_e clk } } -static void dcn3_build_wm_range_table(struct clk_mgr_internal *clk_mgr) +static noinline void dcn3_build_wm_range_table(struct clk_mgr_internal *clk_mgr) { /* defaults */ double pstate_latency_us = clk_mgr->base.ctx->dc->dml.soc.dram_clock_change_latency_us; @@ -201,7 +201,9 @@ void dcn3_init_clocks(struct clk_mgr *clk_mgr_base) clk_mgr_base->funcs->get_memclk_states_from_smu(clk_mgr_base); /* WM range table */ + DC_FP_START(); dcn3_build_wm_range_table(clk_mgr); + DC_FP_END(); } static int dcn30_get_vco_frequency_from_reg(struct clk_mgr_internal *clk_mgr) @@ -416,8 +418,10 @@ static void dcn3_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base) clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1; /* Refresh bounding box */ + DC_FP_START(); clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box( clk_mgr->base.ctx->dc, clk_mgr_base->bw_params); + DC_FP_END(); } static bool dcn3_are_clock_states_equal(struct dc_clocks *a, diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 36a9e9c84ed4480ad7019cb215eb1586639fa996..605ef1bd8cbced260c25ec6955f9583bd0db3900 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -734,12 +734,15 @@ static bool dc_construct(struct dc *dc, goto fail; #ifdef CONFIG_DRM_AMD_DC_DCN3_0 dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present; -#endif dc->debug.force_ignore_link_settings = init_params->force_ignore_link_settings; - if (dc->res_pool->funcs->update_bw_bounding_box) + if (dc->res_pool->funcs->update_bw_bounding_box) { + DC_FP_START(); dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params); + DC_FP_END(); + } +#endif /* Creation of current_state must occur after dc->dml * is initialized in dc_create_resource_pool because diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile index 5fcaf78334ff9a96a24948422733a310bbe08fd2..8651a70737f338a9bc06f2b88e21c61904d1199d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile @@ -17,6 +17,10 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/dcn20/dcn20_resource.o := -mhard-float -maltivec endif +ifdef CONFIG_LOONGARCH +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn20/dcn20_resource.o := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile index 07684d3e375abdceafeb2c43c82259f35a61fb32..255a6edf14a6dadf54e1bd8ec70a5abc57f6706f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile @@ -13,6 +13,10 @@ ifdef CONFIG_PPC64 CFLAGS_$(AMDDALPATH)/dc/dcn21/dcn21_resource.o := -mhard-float -maltivec endif +ifdef CONFIG_LOONGARCH +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn21/dcn21_resource.o := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile index bd2a068f986354ee7c9f5d1730b821d9f86cbfbf..2a1a7e6ac1b32a028b690c12988dd82dbcf2bba4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile @@ -46,12 +46,18 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o := -mgeneral-regs-only CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -mgeneral-regs-only endif +ifdef CONFIG_LOONGARCH +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o := -msoft-float +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 endif endif +ifdef CONFIG_X86 ifdef IS_OLD_GCC # Stack alignment mismatch, proceed with caution. # GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3 @@ -62,6 +68,7 @@ else CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o += -msse2 CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o += -msse2 endif +endif AMD_DAL_DCN30 = $(addprefix $(AMDDALPATH)/dc/dcn30/,$(DCN30)) diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 32993ce24a585e0837e3539a46c5ad16b5ed89fb..ecdb6d033bad3e12be0653d88a0b5c2b72bda024 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -1466,7 +1466,7 @@ int dcn30_populate_dml_pipes_from_context( } void dcn30_populate_dml_writeback_from_context( - struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes) + struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes) { int pipe_cnt, i, j; double max_calc_writeback_dispclk; @@ -1940,7 +1940,7 @@ static struct pipe_ctx *dcn30_find_split_pipe( return pipe; } -static bool dcn30_internal_validate_bw( +static noinline bool dcn30_internal_validate_bw( struct dc *dc, struct dc_state *context, display_e2e_pipe_params_st *pipes, @@ -2202,7 +2202,19 @@ static bool dcn30_internal_validate_bw( return out; } -void dcn30_calculate_wm_and_dlg( +/* + * This must be noinline to ensure anything that deals with FP registers + * is contained within this call; previously our compiling with hard-float + * would result in fp instructions being emitted outside of the boundaries + * of the DC_FP_START/END macros, which makes sense as the compiler has no + * idea about what is wrapped and what is not + * + * This is largely just a workaround to avoid breakage introduced with 5.6, + * ideally all fp-using code should be moved into its own file, only that + * should be compiled with hard-float, and all code exported from there + * should be strictly wrapped with DC_FP_START/END + */ +static noinline void dcn30_calculate_wm_and_dlg_fp( struct dc *dc, struct dc_state *context, display_e2e_pipe_params_st *pipes, int pipe_cnt, @@ -2358,6 +2370,17 @@ void dcn30_calculate_wm_and_dlg( dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us; } +void dcn30_calculate_wm_and_dlg( + struct dc *dc, struct dc_state *context, + display_e2e_pipe_params_st *pipes, + int pipe_cnt, + int vlevel) +{ + DC_FP_START(); + dcn30_calculate_wm_and_dlg_fp(dc, context, pipes, pipe_cnt, vlevel); + DC_FP_END(); +} + bool dcn30_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate) @@ -2373,7 +2396,9 @@ bool dcn30_validate_bandwidth(struct dc *dc, BW_VAL_TRACE_COUNT(); + DC_FP_START(); out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate); + DC_FP_END(); if (pipe_cnt == 0) goto validate_out; @@ -2409,7 +2434,7 @@ bool dcn30_validate_bandwidth(struct dc *dc, return out; } -static void get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, +static noinline void get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, unsigned int *optimal_dcfclk, unsigned int *optimal_fclk) { @@ -2497,8 +2522,10 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params // Calculate optimal dcfclk for each uclk for (i = 0; i < num_uclk_states; i++) { + DC_FP_START(); get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, &optimal_dcfclk_for_uclk[i], NULL); + DC_FP_END(); if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) { optimal_dcfclk_for_uclk[i] = bw_params->clk_table.entries[0].dcfclk_mhz; } @@ -2566,11 +2593,6 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params if (dc->current_state) dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); } - - /* re-init DML with updated bb */ - dml_init_instance(&dc->dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); - if (dc->current_state) - dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30); } static const struct resource_funcs dcn30_res_pool_funcs = { @@ -2602,6 +2624,8 @@ static bool dcn30_resource_construct( struct dc_context *ctx = dc->ctx; struct irq_service_init_data init_data; + DC_FP_START(); + ctx->dc_bios->regs = &bios_regs; pool->base.res_cap = &res_cap_dcn3; @@ -2866,10 +2890,13 @@ static bool dcn30_resource_construct( dc->cap_funcs = cap_funcs; + DC_FP_END(); + return true; create_fail: + DC_FP_END(); dcn30_resource_destruct(pool); return false; diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 417331438c306195914873dd653072efae1ed989..1cf99c828358db8f6df8f236434daa2431005d72 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -33,6 +33,11 @@ ifdef CONFIG_PPC64 dml_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_LOONGARCH +dml_ccflags := -mfpu=64 +dml_rcflags := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -60,13 +65,24 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20v2.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn21/display_mode_vba_21.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn21/display_rq_dlg_calc_21.o := $(dml_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20v2.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn21/display_mode_vba_21.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn21/display_rq_dlg_calc_21.o := $(dml_rcflags) endif ifdef CONFIG_DRM_AMD_DC_DCN3_0 CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_mode_vba_30.o := $(dml_ccflags) -Wframe-larger-than=2048 CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn30/display_mode_vba_30.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_rcflags) endif CFLAGS_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/display_rq_dlg_helpers.o := $(dml_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_rcflags) DML = display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o \ diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile index ea29cf95d470b874eb214391865ff2ead9b3f0e0..02ff495d57a9c8e5e1882775d7f8e485932ecb62 100644 --- a/drivers/gpu/drm/amd/display/dc/dsc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile @@ -10,6 +10,10 @@ ifdef CONFIG_PPC64 dsc_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_LOONGARCH +dsc_rcflags := -msoft-float +endif + ifdef CONFIG_CC_IS_GCC ifeq ($(call cc-ifversion, -lt, 0701, y), y) IS_OLD_GCC = 1 @@ -28,6 +32,7 @@ endif endif CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_ccflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_rcflags) DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h index 126c2f3a4dd3b289b4ab871d8af5a8b48c42e70b..eb7d0021c86e0a2a9d0976a33044a809e40e6626 100644 --- a/drivers/gpu/drm/amd/display/dc/os_types.h +++ b/drivers/gpu/drm/amd/display/dc/os_types.h @@ -55,6 +55,10 @@ #include #define DC_FP_START() kernel_fpu_begin() #define DC_FP_END() kernel_fpu_end() +#elif defined(CONFIG_LOONGARCH) +#include +#define DC_FP_START() kernel_fpu_begin() +#define DC_FP_END() kernel_fpu_end() #elif defined(CONFIG_PPC64) #include #include diff --git a/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c index c8a5a5698edd907634b15174a15b59d715f742e1..6eb6f05c11367f132d8e394eabbf7133e83fa4b3 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/kv_dpm.c @@ -2733,10 +2733,8 @@ static int kv_parse_power_table(struct amdgpu_device *adev) non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) &non_clock_info_array->nonClockInfo[non_clock_array_index]; ps = kzalloc(sizeof(struct kv_ps), GFP_KERNEL); - if (ps == NULL) { - kfree(adev->pm.dpm.ps); + if (ps == NULL) return -ENOMEM; - } adev->pm.dpm.ps[i].ps_priv = ps; k = 0; idx = (u8 *)&power_state->v2.clockInfoIndex[0]; diff --git a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c index d6544a6dabc7c8224dd371b11722128c668f6123..6f0653c81f8fbbcf131951c3716e0b37b8e4414e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c @@ -7349,10 +7349,9 @@ static int si_dpm_init(struct amdgpu_device *adev) kcalloc(4, sizeof(struct amdgpu_clock_voltage_dependency_entry), GFP_KERNEL); - if (!adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) { - amdgpu_free_extended_power_table(adev); + if (!adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) return -ENOMEM; - } + adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4; adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0; adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0; diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c index e941c113259848f0c41ecd25cc9386c9fa50a514..eebb2dece768ba5aa333fe6f4632701a5796a74c 100644 --- a/drivers/gpu/drm/bridge/nxp-ptn3460.c +++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c @@ -54,13 +54,13 @@ static int ptn3460_read_bytes(struct ptn3460_bridge *ptn_bridge, char addr, int ret; ret = i2c_master_send(ptn_bridge->client, &addr, 1); - if (ret <= 0) { + if (ret < 0) { DRM_ERROR("Failed to send i2c command, ret=%d\n", ret); return ret; } ret = i2c_master_recv(ptn_bridge->client, buf, len); - if (ret <= 0) { + if (ret < 0) { DRM_ERROR("Failed to recv i2c data, ret=%d\n", ret); return ret; } @@ -78,7 +78,7 @@ static int ptn3460_write_byte(struct ptn3460_bridge *ptn_bridge, char addr, buf[1] = val; ret = i2c_master_send(ptn_bridge->client, buf, ARRAY_SIZE(buf)); - if (ret <= 0) { + if (ret < 0) { DRM_ERROR("Failed to send i2c command, ret=%d\n", ret); return ret; } diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c index 1a636963378947cc247ee46c6573ecf978362060..ad10d640fa94a76369dc2eb773eb166f84f2b4c8 100644 --- a/drivers/gpu/drm/drm_vm.c +++ b/drivers/gpu/drm/drm_vm.c @@ -74,7 +74,7 @@ static pgprot_t drm_io_prot(struct drm_local_map *map, tmp = pgprot_decrypted(tmp); #if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \ - defined(__mips__) + defined(__mips__) || defined(__loongarch__) if (map->type == _DRM_REGISTERS && !(map->flags & _DRM_WRITE_COMBINING)) tmp = pgprot_noncached(tmp); else diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c index c277d2fc50c66dcf961f486ae7f4ab0f9d38db60..e43dfea09527fed8c0dd6d0031c63d4a385d7282 100644 --- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c @@ -318,9 +318,9 @@ static void decon_win_set_bldmod(struct decon_context *ctx, unsigned int win, static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win, struct drm_framebuffer *fb) { - struct exynos_drm_plane plane = ctx->planes[win]; + struct exynos_drm_plane *plane = &ctx->planes[win]; struct exynos_drm_plane_state *state = - to_exynos_plane_state(plane.base.state); + to_exynos_plane_state(plane->base.state); unsigned int alpha = state->base.alpha; unsigned int pixel_alpha; unsigned long val; diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c index bb67cad8371f03da95d878e334e32b1cff2b10e9..c045330f9c48fa05e766725ebc40338dc2459a27 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c @@ -637,9 +637,9 @@ static void fimd_win_set_bldmod(struct fimd_context *ctx, unsigned int win, static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win, struct drm_framebuffer *fb, int width) { - struct exynos_drm_plane plane = ctx->planes[win]; + struct exynos_drm_plane *plane = &ctx->planes[win]; struct exynos_drm_plane_state *state = - to_exynos_plane_state(plane.base.state); + to_exynos_plane_state(plane->base.state); uint32_t pixel_format = fb->format->format; unsigned int alpha = state->base.alpha; u32 val = WINCONx_ENWIN; diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c index 45e9aee8366a8f5bfc14fd60c6301424a5603f17..bcf830c5b8ea93f2157594484f3ebd9fb7e74aed 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c +++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c @@ -1344,7 +1344,7 @@ static int __maybe_unused gsc_runtime_resume(struct device *dev) for (i = 0; i < ctx->num_clocks; i++) { ret = clk_prepare_enable(ctx->clocks[i]); if (ret) { - while (--i > 0) + while (--i >= 0) clk_disable_unprepare(ctx->clocks[i]); return ret; } diff --git a/drivers/gpu/drm/loongson/Kconfig b/drivers/gpu/drm/loongson/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..0c5f22703b6a981c8fc7a04010ad96d43e98187f --- /dev/null +++ b/drivers/gpu/drm/loongson/Kconfig @@ -0,0 +1,19 @@ + +config DRM_LOONGSON + tristate "DRM support for LS7A Display Controller" + depends on DRM && PCI + select FB_CFB_FILLRECT + select FB_CFB_COPYAREA + select FB_CFB_IMAGEBLIT + select DRM_KMS_HELPER + select DRM_KMS_FB_HELPER + select DRM_GEM_CMA_HELPER + select DRM_KMS_CMA_HELPER + default n + help + Support the display controllers found on the Loongson's LS7A + bridge. + + This driver provides no built-in acceleration; acceleration is + performed by Vivante GC1000. This driver provides kernel mode + setting and buffer management to userspace. diff --git a/drivers/gpu/drm/loongson/Makefile b/drivers/gpu/drm/loongson/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a5272137848c52b7d911acc9eb184acbdbe0b9d4 --- /dev/null +++ b/drivers/gpu/drm/loongson/Makefile @@ -0,0 +1,16 @@ +# +# Makefile for the drm device driver. This driver provides support for the +# Direct Rendering Infrastructure (DRI) +# + +ccflags-y := -Iinclude/drm +loongson-y := loongson_drv.o \ + loongson_i2c.o \ + loongson_irq.o \ + loongson_crtc.o \ + loongson_encoder.o \ + loongson_connector.o \ + loongson_cursor.o \ + loongson_vbios.o + +obj-$(CONFIG_DRM_LOONGSON) += loongson.o diff --git a/drivers/gpu/drm/loongson/loongson_connector.c b/drivers/gpu/drm/loongson/loongson_connector.c new file mode 100644 index 0000000000000000000000000000000000000000..721336709a6b90e4f8303658d0fcac61dd8c30d5 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_connector.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "loongson_drv.h" + +/** + * loongson_connector_best_encoder + * + * @connector: point to the drm_connector structure + * + * Select the best encoder for the given connector. Used by both the helpers in + * drm_atomic_helper_check_modeset() and legacy CRTC helpers + */ +static struct drm_encoder *loongson_connector_best_encoder(struct drm_connector + *connector) +{ + struct drm_encoder *encoder; + + /* There is only one encoder per connector */ + drm_connector_for_each_possible_encoder(connector, encoder) + return encoder; + + return NULL; +} + +/** + * loongson_get_modes + * + * @connetcor: central DRM connector control structure + * + * Fill in all modes currently valid for the sink into the connector->probed_modes list. + * It should also update the EDID property by calling drm_connector_update_edid_property(). + */ +static int loongson_get_modes(struct drm_connector *connector) +{ + int id, ret = 0; + enum loongson_edid_method ledid_method; + struct edid *edid = NULL; + struct loongson_connector *lconnector = to_loongson_connector(connector); + struct i2c_adapter *adapter = lconnector->i2c->adapter; + + id = drm_connector_index(connector); + + ledid_method = lconnector->edid_method; + switch (ledid_method) { + case via_i2c: + case via_encoder: + default: + edid = drm_get_edid(connector, adapter); + break; + case via_vbios: + edid = kmalloc(EDID_LENGTH * 2, GFP_KERNEL); + memcpy(edid, lconnector->vbios_edid, EDID_LENGTH * 2); + } + + if (edid) { + drm_connector_update_edid_property(connector, edid); + ret = drm_add_edid_modes(connector, edid); + kfree(edid); + } else { + ret += drm_add_modes_noedid(connector, 1920, 1080); + drm_set_preferred_mode(connector, 1024, 768); + } + + return ret; +} + +static bool is_connected(struct loongson_connector *lconnector) +{ + unsigned char start = 0x0; + struct i2c_adapter *adapter; + struct i2c_msg msgs = { + .addr = DDC_ADDR, + .len = 1, + .flags = 0, + .buf = &start, + }; + + if (!lconnector->i2c) + return false; + + adapter = lconnector->i2c->adapter; + if (i2c_transfer(adapter, &msgs, 1) < 1) { + DRM_DEBUG_KMS("display-%d not connect\n", lconnector->id); + return false; + } + + return true; +} + +/** + * loongson_connector_detect + * + * @connector: point to drm_connector + * @force: bool + * + * Check to see if anything is attached to the connector. + * The parameter force is set to false whilst polling, + * true when checking the connector due to a user request + */ +static enum drm_connector_status loongson_connector_detect(struct drm_connector + *connector, bool force) +{ + int r; + enum drm_connector_status ret = connector_status_connected; + struct loongson_connector *lconnector = to_loongson_connector(connector); + + DRM_DEBUG("loongson_connector_detect connector_id=%d, ledid_method=%d\n", + drm_connector_index(connector), lconnector->edid_method); + + if (lconnector->edid_method != via_vbios) { + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + + if (is_connected(lconnector)) + ret = connector_status_connected; + else + ret = connector_status_disconnected; + + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + } + + return ret; +} + +/** + * These provide the minimum set of functions required to handle a connector + * + * Helper operations for connectors.These functions are used + * by the atomic and legacy modeset helpers and by the probe helpers. + */ +static const struct drm_connector_helper_funcs loongson_connector_helper_funcs = { + .get_modes = loongson_get_modes, + .best_encoder = loongson_connector_best_encoder, +}; + +/** + * These provide the minimum set of functions required to handle a connector + * + * Control connectors on a given device. + * The functions below allow the core DRM code to control connectors, + * enumerate available modes and so on. + */ +static const struct drm_connector_funcs loongson_connector_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = loongson_connector_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = drm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static const unsigned short normal_i2c[] = { 0x50, I2C_CLIENT_END }; + + +/** + * loongson_connector_init + * + * @dev: drm device + * @connector_id: + * + * Vga is the interface between host and monitor + * This function is to init vga + */ +struct drm_connector *loongson_connector_init(struct drm_device *dev, unsigned int index) +{ + struct i2c_adapter *adapter; + struct i2c_client *ddc_client; + struct drm_connector *connector; + struct loongson_encoder *loongson_encoder; + struct loongson_connector *loongson_connector; + struct loongson_drm_device *ldev = (struct loongson_drm_device*)dev->dev_private; + + const struct i2c_board_info ddc_info = { + .type = "ddc-dev", + .addr = DDC_ADDR, + .flags = I2C_CLASS_DDC, + }; + + loongson_encoder = ldev->mode_info[index].encoder; + adapter = loongson_encoder->i2c->adapter; + ddc_client = i2c_new_client_device(adapter, &ddc_info); + if (IS_ERR(ddc_client)) { + i2c_del_adapter(adapter); + DRM_ERROR("Failed to create standard ddc client\n"); + return NULL; + } + + loongson_connector = kzalloc(sizeof(struct loongson_connector), GFP_KERNEL); + if (!loongson_connector) + return NULL; + + ldev->connector_active0 = 0; + ldev->connector_active1 = 0; + loongson_connector->id = index; + loongson_connector->ldev = ldev; + loongson_connector->type = get_connector_type(ldev, index); + loongson_connector->i2c_id = get_connector_i2cid(ldev, index); + loongson_connector->hotplug = get_hotplug_mode(ldev, index); + loongson_connector->edid_method = get_edid_method(ldev, index); + if (loongson_connector->edid_method == via_vbios) + loongson_connector->vbios_edid = get_vbios_edid(ldev, index); + + loongson_connector->i2c = &ldev->i2c_bus[index]; + if (!loongson_connector->i2c) + DRM_INFO("connector-%d match i2c-%d err\n", index, + loongson_connector->i2c_id); + + connector = &loongson_connector->base; + + drm_connector_helper_add(connector, &loongson_connector_helper_funcs); + + drm_connector_init(dev, connector, + &loongson_connector_funcs, loongson_connector->type); + + drm_connector_register(connector); + + return connector; +} diff --git a/drivers/gpu/drm/loongson/loongson_crtc.c b/drivers/gpu/drm/loongson/loongson_crtc.c new file mode 100644 index 0000000000000000000000000000000000000000..a9972a9a1a687423b40e97901ab043d101f21e61 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_crtc.c @@ -0,0 +1,509 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * Jiaxun Yang + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "loongson_drv.h" + +/** + * This file contains setup code for the CRTC + */ + +DEFINE_SPINLOCK(loongson_crtc_lock); + +static int loongson_crtc_enable_vblank(struct drm_crtc *crtc) +{ + struct loongson_crtc *lcrtc = to_loongson_crtc(crtc); + struct loongson_drm_device *ldev = lcrtc->ldev; + + if(lcrtc->crtc_id == 0) { + ldev->int_reg |= (BIT(INT_DVO0_FB_END) << 16); + } else { + ldev->int_reg |= (BIT(INT_DVO1_FB_END) << 16); + } + + spin_lock(&loongson_reglock); + writel(ldev->int_reg, ldev->mmio + FB_INT_REG); + spin_unlock(&loongson_reglock); + + return 0; +} + +static void loongson_crtc_disable_vblank(struct drm_crtc *crtc) +{ + struct loongson_crtc *lcrtc = to_loongson_crtc(crtc); + struct loongson_drm_device *ldev = lcrtc->ldev; + + + if(lcrtc->crtc_id == 0) { + ldev->int_reg &= (~BIT(INT_DVO0_FB_END) << 16); + } else { + ldev->int_reg &= (~BIT(INT_DVO1_FB_END) << 16); + } + + spin_lock(&loongson_reglock); + writel(ldev->int_reg, ldev->mmio + FB_INT_REG); + spin_unlock(&loongson_reglock); +} + +#define PLL_REF_CLK_MHZ 100 +#define PCLK_PRECISION_INDICATOR 10000 + +/** + * cal_freq + * + * @pixclock: unsigned int + * @pll_config: point to the pix_pll structure + * + * Calculate frequency + */ +static unsigned int cal_freq(unsigned int pixclock, struct pix_pll *pll_config) +{ + int i, j, loopc_offset; + unsigned int refc_set[] = {4, 5, 3}; + unsigned int prec_set[] = {1, 5, 10, 50, 100}; /*in 1/PCLK_PRECISION_INDICATOR*/ + unsigned int pstdiv, loopc, refc; + unsigned int precision_req, precision; + unsigned int loopc_min, loopc_max, loopc_mid; + unsigned long long real_dvo, req_dvo; + + /*try precision from high to low*/ + for (j = 0; j < sizeof(prec_set)/sizeof(int); j++){ + precision_req = prec_set[j]; + + /*try each refc*/ + for (i = 0; i < sizeof(refc_set)/sizeof(int); i++) { + refc = refc_set[i]; + loopc_min = (1200 / PLL_REF_CLK_MHZ) * refc; /*1200 / (PLL_REF_CLK_MHZ / refc)*/ + loopc_max = (3200 / PLL_REF_CLK_MHZ) * refc; /*3200 / (PLL_REF_CLK_MHZ / refc)*/ + loopc_mid = (2200 / PLL_REF_CLK_MHZ) * refc; /*(loopc_min + loopc_max) / 2;*/ + loopc_offset = -1; + + /*try each loopc*/ + for (loopc = loopc_mid; (loopc <= loopc_max) && (loopc >= loopc_min); loopc += loopc_offset) { + if(loopc_offset < 0) + loopc_offset = -loopc_offset; + else + loopc_offset = -(loopc_offset+1); + + pstdiv = loopc * PLL_REF_CLK_MHZ * 1000 / refc / pixclock; + if((pstdiv > 127) || (pstdiv < 1)) + continue; + + /*real_freq is float type which is not available, but read_freq * pstdiv is available.*/ + req_dvo = (pixclock * pstdiv); + real_dvo = (loopc * PLL_REF_CLK_MHZ * 1000 / refc); + precision = abs(real_dvo * PCLK_PRECISION_INDICATOR / req_dvo - PCLK_PRECISION_INDICATOR); + + if(precision < precision_req){ + pll_config->l2_div = pstdiv; + pll_config->l1_loopc = loopc; + pll_config->l1_frefc = refc; + if(j > 1) + printk("Warning: PIX clock precision degraded to %d / %d\n", precision_req, PCLK_PRECISION_INDICATOR); + return 1; + } + } + } + } + return 0; +} + +/** + * config_pll + * + * @pll_base: represent a long type + * @pll_cfg: point to the pix_pll srtucture + * + * Config pll apply to ls7a + */ +static void config_pll(void *pll_base, struct pix_pll *pll_cfg) +{ + unsigned long val; + + /* clear sel_pll_out0 */ + val = readl(pll_base + 0x4); + val &= ~(1UL << 8); + writel(val, pll_base + 0x4); + /* set pll_pd */ + val = readl(pll_base + 0x4); + val |= (1UL << 13); + writel(val, pll_base + 0x4); + /* clear set_pll_param */ + val = readl(pll_base + 0x4); + val &= ~(1UL << 11); + writel(val, pll_base + 0x4); + /* clear old value & config new value */ + val = readl(pll_base + 0x4); + val &= ~(0x7fUL << 0); + val |= (pll_cfg->l1_frefc << 0); /* refc */ + writel(val, pll_base + 0x4); + val = readl(pll_base + 0x0); + val &= ~(0x7fUL << 0); + val |= (pll_cfg->l2_div << 0); /* div */ + val &= ~(0x1ffUL << 21); + val |= (pll_cfg->l1_loopc << 21);/* loopc */ + writel(val, pll_base + 0x0); + /* set set_pll_param */ + val = readl(pll_base + 0x4); + val |= (1UL << 11); + writel(val, pll_base + 0x4); + /* clear pll_pd */ + val = readl(pll_base + 0x4); + val &= ~(1UL << 13); + writel(val, pll_base + 0x4); + /* wait pll lock */ + while(!(readl(pll_base + 0x4) & 0x80)) + cpu_relax(); + /* set sel_pll_out0 */ + val = readl(pll_base + 0x4); + val |= (1UL << 8); + writel(val, pll_base + 0x4); +} + +static void loongson_config_pll(int id, unsigned int pix_freq) +{ + unsigned int out; + struct pix_pll pll_cfg; + + out = cal_freq(pix_freq, &pll_cfg); + if (id == 0) + config_pll(LS7A_PIX0_PLL, &pll_cfg); + else + config_pll(LS7A_PIX1_PLL, &pll_cfg); +} + +/** + * These provide the minimum set of functions required to handle a CRTC + * Each driver is responsible for filling out this structure at startup time + * + * The drm_crtc_funcs structure is the central CRTC management structure + * in the DRM. Each CRTC controls one or more connectors + */ +static const struct drm_crtc_funcs loongson_swcursor_crtc_funcs = { + .destroy = drm_crtc_cleanup, + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = loongson_crtc_enable_vblank, + .disable_vblank = loongson_crtc_disable_vblank, +}; + +static const struct drm_crtc_funcs loongson_hwcursor_crtc_funcs = { + .cursor_set2 = loongson_crtc_cursor_set2, + .cursor_move = loongson_crtc_cursor_move, + .destroy = drm_crtc_cleanup, + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = loongson_crtc_enable_vblank, + .disable_vblank = loongson_crtc_disable_vblank, +}; + +static const uint32_t loongson_formats[] = { + DRM_FORMAT_RGB565, + DRM_FORMAT_RGB888, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, +}; + +static const uint64_t loongson_format_modifiers[] = { + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + +static enum drm_mode_status loongson_crtc_mode_valid(struct drm_crtc *crtc, + const struct drm_display_mode *mode) +{ + int id = crtc->index; + struct drm_device *dev = crtc->dev; + struct loongson_drm_device *ldev = (struct loongson_drm_device*)dev->dev_private; + + if (mode->hdisplay > get_crtc_max_width(ldev, id)) + return MODE_BAD; + if (mode->vdisplay > get_crtc_max_height(ldev, id)) + return MODE_BAD; + if (ldev->num_crtc == 1) { + if (mode->hdisplay % 16) + return MODE_BAD; + } else { + if (mode->hdisplay % 64) + return MODE_BAD; + } + + return MODE_OK; +} + +u32 crtc_read(struct loongson_crtc *lcrtc, u32 offset) +{ + struct loongson_drm_device *ldev = lcrtc->ldev; + return readl(ldev->mmio + offset + (lcrtc->crtc_id * CRTC_REG_OFFSET)); +} + +void crtc_write(struct loongson_crtc *lcrtc, u32 offset, u32 val) +{ + struct loongson_drm_device *ldev = lcrtc->ldev; + writel(val, ldev->mmio + offset + (lcrtc->crtc_id * CRTC_REG_OFFSET)); +} + +static void loongson_crtc_mode_set_nofb(struct drm_crtc *crtc) +{ + unsigned int hr, hss, hse, hfl; + unsigned int vr, vss, vse, vfl; + unsigned int pix_freq; + unsigned long flags; + struct loongson_crtc *lcrtc = to_loongson_crtc(crtc); + struct drm_display_mode *mode = &crtc->state->adjusted_mode; + + hr = mode->hdisplay; + hss = mode->hsync_start; + hse = mode->hsync_end; + hfl = mode->htotal; + + vr = mode->vdisplay; + vss = mode->vsync_start; + vse = mode->vsync_end; + vfl = mode->vtotal; + + pix_freq = mode->clock; + + DRM_DEBUG("crtc_id = %d, hr = %d, hss = %d, hse = %d, hfl = %d, vr = %d, vss = %d, vse = %d, vfl = %d, pix_freq = %d,\n", + lcrtc->crtc_id, hr, hss, hse, hfl, vr, vss, vse, vfl, pix_freq); + + spin_lock_irqsave(&loongson_reglock, flags); + crtc_write(lcrtc, FB_DITCFG_DVO_REG, 0); + crtc_write(lcrtc, FB_DITTAB_LO_DVO_REG, 0); + crtc_write(lcrtc, FB_DITTAB_HI_DVO_REG, 0); + crtc_write(lcrtc, FB_PANCFG_DVO_REG, 0x80001311); + crtc_write(lcrtc, FB_PANTIM_DVO_REG, 0); + + crtc_write(lcrtc, FB_HDISPLAY_DVO_REG, (mode->crtc_htotal << 16) | mode->crtc_hdisplay); + crtc_write(lcrtc, FB_HSYNC_DVO_REG, 0x40000000 | (mode->crtc_hsync_end << 16) | mode->crtc_hsync_start); + + crtc_write(lcrtc, FB_VDISPLAY_DVO_REG, (mode->crtc_vtotal << 16) | mode->crtc_vdisplay); + crtc_write(lcrtc, FB_VSYNC_DVO_REG, 0x40000000 | (mode->crtc_vsync_end << 16) | mode->crtc_vsync_start); + + crtc_write(lcrtc, FB_STRI_DVO_REG, (crtc->primary->state->fb->pitches[0] + 255) & ~255); + + DRM_DEBUG("Stride: %x\n",(crtc->primary->state->fb->pitches[0] + 255) & ~255); + + switch (crtc->primary->state->fb->format->format) { + case DRM_FORMAT_RGB565: + lcrtc->cfg_reg |= 0x3; + crtc_write(lcrtc, FB_CFG_DVO_REG, lcrtc->cfg_reg); + break; + case DRM_FORMAT_RGB888: + default: + lcrtc->cfg_reg |= 0x4; + crtc_write(lcrtc, FB_CFG_DVO_REG, lcrtc->cfg_reg); + break; + } + spin_unlock_irqrestore(&loongson_reglock, flags); + + loongson_config_pll(lcrtc->crtc_id, mode->clock); +} + +static void loongson_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_crtc_state *old_state) +{ + unsigned long flags; + struct loongson_crtc *lcrtc = to_loongson_crtc(crtc); + + if (lcrtc->cfg_reg & CFG_ENABLE) + goto vblank_on; + + lcrtc->cfg_reg |= CFG_ENABLE; + spin_lock_irqsave(&loongson_reglock, flags); + crtc_write(lcrtc, FB_CFG_DVO_REG, lcrtc->cfg_reg); + spin_unlock_irqrestore(&loongson_reglock, flags); + +vblank_on: + drm_crtc_vblank_on(crtc); +} + +static void loongson_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_crtc_state *old_state) +{ + unsigned long flags; + struct loongson_crtc *lcrtc = to_loongson_crtc(crtc); + + lcrtc->cfg_reg &= ~CFG_ENABLE; + spin_lock_irqsave(&loongson_reglock, flags); + crtc_write(lcrtc, FB_CFG_DVO_REG, lcrtc->cfg_reg); + spin_unlock_irqrestore(&loongson_reglock, flags); + + spin_lock_irq(&crtc->dev->event_lock); + if (crtc->state->event) { + drm_crtc_send_vblank_event(crtc, crtc->state->event); + crtc->state->event = NULL; + } + spin_unlock_irq(&crtc->dev->event_lock); + + drm_crtc_vblank_off(crtc); +} + +static void loongson_crtc_atomic_flush(struct drm_crtc *crtc, + struct drm_crtc_state *old_crtc_state) +{ + struct drm_pending_vblank_event *event = crtc->state->event; + + if (event) { + crtc->state->event = NULL; + + spin_lock_irq(&crtc->dev->event_lock); + if (drm_crtc_vblank_get(crtc) == 0) + drm_crtc_arm_vblank_event(crtc, event); + else + drm_crtc_send_vblank_event(crtc, event); + spin_unlock_irq(&crtc->dev->event_lock); + } +} + +static void loongson_plane_atomic_update(struct drm_plane *plane, + struct drm_plane_state *old_state) +{ + int id, clonemode; + unsigned int pitch; + unsigned long flags; + struct loongson_crtc *lcrtc; + struct loongson_drm_device *ldev; + struct drm_plane_state *state = plane->state; + + if (!state->crtc || !state->fb) + return; + + pitch = state->fb->pitches[0]; + lcrtc = to_loongson_crtc(state->crtc); + ldev = lcrtc->ldev; + id = lcrtc->crtc_id; + clonemode = clone_mode(ldev); + + /* CRTC1 cloned from CRTC0 in clone mode */ + if (clonemode) + ldev->lcrtc[1].cfg_reg |= CFG_PANELSWITCH; + else + ldev->lcrtc[1].cfg_reg &= ~CFG_PANELSWITCH; + + spin_lock_irqsave(&loongson_reglock, flags); + crtc_write(lcrtc, FB_STRI_DVO_REG, (pitch + 255) & ~255); + if (crtc_read(lcrtc, FB_CFG_DVO_REG) & CFG_FBNUM) + crtc_write(lcrtc, FB_ADDR0_DVO_REG, drm_fb_cma_get_gem_addr(state->fb, state, 0)); + else + crtc_write(lcrtc, FB_ADDR1_DVO_REG, drm_fb_cma_get_gem_addr(state->fb, state, 0)); + + lcrtc->cfg_reg |= CFG_ENABLE; + crtc_write(lcrtc, FB_CFG_DVO_REG, lcrtc->cfg_reg | CFG_FBSWITCH); + if (clonemode) { + if (id) + crtc_write(&ldev->lcrtc[0], FB_CFG_DVO_REG, ldev->lcrtc[0].cfg_reg | CFG_ENABLE); + else + crtc_write(&ldev->lcrtc[1], FB_CFG_DVO_REG, ldev->lcrtc[1].cfg_reg | CFG_ENABLE); + } + spin_unlock_irqrestore(&loongson_reglock, flags); + + udelay(2500); +} + +/** + * These provide the minimum set of functions required to handle a CRTC + * + * The drm_crtc_helper_funcs is a helper operations for CRTC + */ +static const struct drm_crtc_helper_funcs loongson_crtc_helper_funcs = { + .mode_valid = loongson_crtc_mode_valid, + .mode_set_nofb = loongson_crtc_mode_set_nofb, + .atomic_enable = loongson_crtc_atomic_enable, + .atomic_disable = loongson_crtc_atomic_disable, + .atomic_flush = loongson_crtc_atomic_flush, +}; + +static void loongson_plane_destroy(struct drm_plane *plane) +{ + drm_plane_cleanup(plane); +} + +static bool loongson_format_mod_supported(struct drm_plane *plane, + uint32_t format, uint64_t modifier) +{ + return (modifier == DRM_FORMAT_MOD_LINEAR); +} + +static const struct drm_plane_funcs loongson_plane_funcs = { + .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, + .destroy = loongson_plane_destroy, + .disable_plane = drm_atomic_helper_disable_plane, + .reset = drm_atomic_helper_plane_reset, + .update_plane = drm_atomic_helper_update_plane, + .format_mod_supported = loongson_format_mod_supported, +}; + +static const struct drm_plane_helper_funcs loongson_plane_helper_funcs = { + .atomic_update = loongson_plane_atomic_update, +}; + +/** + * loongosn_crtc_init + * + * @ldev: point to the loongson_drm_device structure + * + * Init CRTC + */ +int loongson_crtc_init(struct loongson_drm_device *ldev) +{ + int i, ret; + extern bool hw_cursor; + + for(i=0;inum_crtc;i++){ + ldev->lcrtc[i].ldev = ldev; + ldev->lcrtc[i].crtc_id = i; + + ldev->lcrtc[i].cfg_reg = CFG_RESET; + ldev->lcrtc[i].primary = devm_kzalloc(ldev->dev->dev, sizeof(*ldev->lcrtc[i].primary), GFP_KERNEL); + if (!ldev->lcrtc[i].primary) + return -ENOMEM; + + ret = drm_universal_plane_init(ldev->dev, ldev->lcrtc[i].primary, BIT(i), &loongson_plane_funcs, + loongson_formats, ARRAY_SIZE(loongson_formats), + loongson_format_modifiers, DRM_PLANE_TYPE_PRIMARY, NULL); + if (ret) + return ret; + + drm_plane_helper_add(ldev->lcrtc[i].primary, &loongson_plane_helper_funcs); + + if (hw_cursor) + ret = drm_crtc_init_with_planes(ldev->dev, &ldev->lcrtc[i].base,ldev->lcrtc[i].primary, + NULL, &loongson_hwcursor_crtc_funcs, NULL); + else + ret = drm_crtc_init_with_planes(ldev->dev, &ldev->lcrtc[i].base,ldev->lcrtc[i].primary, + NULL, &loongson_swcursor_crtc_funcs, NULL); + if (ret) { + loongson_plane_destroy(ldev->lcrtc[i].primary); + return ret; + } + drm_crtc_helper_add(&ldev->lcrtc[i].base, &loongson_crtc_helper_funcs); + } + + return 0; +} diff --git a/drivers/gpu/drm/loongson/loongson_cursor.c b/drivers/gpu/drm/loongson/loongson_cursor.c new file mode 100644 index 0000000000000000000000000000000000000000..2f792539168a0570b5de34563df04c4eba15e28e --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_cursor.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include "loongson_drv.h" + +static void flush_scache_range(void *addr, unsigned long size) +{ + int inc; + unsigned long start, end; + + start = (unsigned long)addr; + end = (unsigned long)addr + size; + inc = cpu_last_level_cache_line_size(); + + for (; start < end; start += inc) + flush_cache_line(Cache_LEAF3, start); +} + +/* + Hide the cursor off screen. We can't disable the cursor hardware because it + takes too long to re-activate and causes momentary corruption +*/ +static void loongson_hide_cursor(struct drm_crtc *crtc) +{ + unsigned long flags; + volatile void __iomem *base; + struct drm_device *dev = crtc->dev; + struct loongson_drm_device *ldev = (struct loongson_drm_device *)dev->dev_private; + struct loongson_crtc *loongson_crtc = to_loongson_crtc(crtc); + unsigned int tmp, crtc_id = loongson_crtc->crtc_id; + + base = ldev->mmio; + tmp = readl(base + FB_CUR_CFG_REG); + tmp &= ~0xff; + if (clone_mode(ldev)) { + spin_lock_irqsave(&loongson_reglock, flags); + writel(tmp | 0x00, base + FB_CUR_CFG_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); + ldev->cursor_showed = false; + } else { + if (ldev->cursor_crtc_id != crtc_id) + return; + + spin_lock_irqsave(&loongson_reglock, flags); + if (crtc_id) { + writel(tmp | 0x10, base + FB_CUR_CFG_REG); + } else { + writel(tmp | 0x00, base + FB_CUR_CFG_REG); + } + spin_unlock_irqrestore(&loongson_reglock, flags); + ldev->cursor_showed = false; + } +} + +static void loongson_show_cursor(struct drm_crtc *crtc) +{ + unsigned long flags; + volatile void __iomem *base; + struct drm_device *dev = crtc->dev; + struct loongson_drm_device *ldev = (struct loongson_drm_device *)dev->dev_private; + struct loongson_crtc *loongson_crtc = to_loongson_crtc(crtc); + unsigned int crtc_id = loongson_crtc->crtc_id; + + base = ldev->mmio; + if (clone_mode(ldev)) { + spin_lock_irqsave(&loongson_reglock, flags); + writel(0x00050202, base + FB_CUR_CFG_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); + ldev->cursor_crtc_id = 0; + ldev->cursor_showed = true; + } else { + if (ldev->cursor_crtc_id == crtc_id) { + spin_lock_irqsave(&loongson_reglock, flags); + if(crtc_id == 0){ + writel(0x00050202, base + FB_CUR_CFG_REG); + }else{ + writel(0x00050212, base + FB_CUR_CFG_REG); + } + spin_unlock_irqrestore(&loongson_reglock, flags); + + ldev->cursor_showed = true; + ldev->cursor_crtc_id = crtc_id; + } + } +} + +int loongson_crtc_cursor_set2(struct drm_crtc *crtc, + struct drm_file *file_priv, + uint32_t handle, + uint32_t width, + uint32_t height, + int32_t hot_x, int32_t hot_y) +{ + u32 gpu_addr; + unsigned long flags; + unsigned int crtc_id; + volatile void __iomem *base; + struct drm_gem_object *obj; + struct drm_device *dev = crtc->dev; + struct loongson_crtc *loongson_crtc = to_loongson_crtc(crtc); + struct loongson_drm_device *ldev = (struct loongson_drm_device *)dev->dev_private; + struct drm_gem_cma_object *cma, *cursor = ldev->cursor; + + base = ldev->mmio; + crtc_id = loongson_crtc->crtc_id; + + if ((width != 32 || height != 32) && handle) { + return -EINVAL; + } + + if (!handle || !file_priv) { + loongson_hide_cursor(crtc); + return 0; + } + + obj = drm_gem_object_lookup(file_priv, handle); + if (!obj) + return -ENOENT; + + cma = to_drm_gem_cma_obj(obj); + + flush_scache_range(cma->vaddr, 32*32*4); + memcpy(cursor->vaddr, cma->vaddr, 32*32*4); + + /* Program gpu address of cursor buffer */ + gpu_addr = ldev->cursor->paddr; + spin_lock_irqsave(&loongson_reglock, flags); + writel(gpu_addr, base + FB_CUR_ADDR_REG); + writel(0x00eeeeee, base + FB_CUR_BACK_REG); + writel(0x00aaaaaa, base + FB_CUR_FORE_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); + + loongson_show_cursor(crtc); + + drm_gem_object_put(obj); + + return 0; +} + +int loongson_crtc_cursor_move(struct drm_crtc *crtc, int x, int y) +{ + unsigned long flags; + unsigned int tmp, crtc_id; + int xorign = 0, yorign = 0; + volatile void __iomem *base; + struct loongson_crtc *loongson_crtc = to_loongson_crtc(crtc); + struct loongson_drm_device *ldev = (struct loongson_drm_device *)crtc->dev->dev_private; + + base = ldev->mmio; + crtc_id = loongson_crtc->crtc_id; + + /* upper edge condition */ + yorign = y + crtc->y; + if (yorign < 0) + y = 0; + + /* left edge conditon */ + xorign = x + crtc->x; + if (xorign < 0) + x = 0; + + /* move from one crtc to another, check which crtc should he shown + * the x or y < 0, it means the cursor it out of current review, + * && xorign/ yorign > 0, it means the cursor is in the framebuffer + * but not in curren review */ + if ((x < 0 && xorign > 0) || (y < 0 && yorign > 0)) { + if(ldev->cursor_crtc_id == crtc_id && !clone_mode(ldev)) + /*the cursor is not show, so hide if the (x,y) is in active crtc*/ + loongson_hide_cursor(crtc); + return 0; + } + + if (x < 0) + x = 0; + if (y < 0) + y = 0; + + tmp = x & 0xffff; + tmp |= y << 16; + spin_lock_irqsave(&loongson_reglock, flags); + writel(tmp, base + FB_CUR_LOC_ADDR_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); + if (ldev->cursor_crtc_id != crtc_id && !clone_mode(ldev)) { + ldev->cursor_crtc_id = crtc_id; + ldev->cursor_showed = false; + } + if (!ldev->cursor_showed) + loongson_show_cursor(crtc); + + return 0; +} diff --git a/drivers/gpu/drm/loongson/loongson_drv.c b/drivers/gpu/drm/loongson/loongson_drv.c new file mode 100644 index 0000000000000000000000000000000000000000..b705045d50e7603ec0c4342912ca4b7ff5f3bf82 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_drv.c @@ -0,0 +1,656 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "loongson_drv.h" + +#define DEVICE_NAME "loongson-drm" +#define DRIVER_NAME "loongson-drm" +#define DRIVER_DESC "Loongson DRM Driver" +#define DRIVER_DATE "20201201" +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 0 +#define DRIVER_PATCHLEVEL 1 + +bool hw_cursor = false; +module_param_named(cursor, hw_cursor, bool, 0600); + +bool poll_connector = false; +module_param_named(poll, poll_connector, bool, 0600); + +DEFINE_SPINLOCK(loongson_reglock); + +/** + * loongson_mode_funcs---basic driver provided mode setting functions + * + * Some global (i.e. not per-CRTC, connector, etc) mode setting functions that + * involve drivers. + */ +static const struct drm_mode_config_funcs loongson_mode_funcs = { + .fb_create = drm_gem_fb_create, + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, + .output_poll_changed = drm_fb_helper_output_poll_changed +}; + +/** + * loongson_drm_device_init ----init drm device + * + * @dev pointer to drm_device structure + * @flags start up flag + * + * RETURN + * drm device init result + */ +static int loongson_drm_device_init(struct drm_device *dev, uint32_t flags) +{ + int ret = 0; + struct loongson_drm_device *ldev = dev->dev_private; + + ldev->num_crtc = 2; + loongson_vbios_init(ldev); + + /*BAR 0 contains registers */ + ldev->mmio_base = pci_resource_start(ldev->dev->pdev, 0); + ldev->mmio_size = pci_resource_len(ldev->dev->pdev, 0); + + ldev->mmio = pcim_iomap(dev->pdev, 0, 0); + if (ldev->mmio == NULL) + return -ENOMEM; + + DRM_INFO("ldev->mmio_base = 0x%llx, ldev->mmio_size = 0x%llx\n", + ldev->mmio_base, ldev->mmio_size); + + if (!devm_request_mem_region(ldev->dev->dev, ldev->mmio_base, ldev->mmio_size, + "loongson_drmfb_mmio")) { + DRM_ERROR("Can't reserve mmio registers\n"); + return -ENOMEM; + } + + ret = loongson_gpio_init(ldev); + if (ret < 0) + DRM_ERROR("Failed to initialize dc gpios\n"); + + return ret; +} + +/** + * loongson_modeset_init --- init kernel mode setting + * + * @ldev: pointer to loongson_drm_device structure + * + * RETURN + * return init result + */ +int loongson_modeset_init(struct loongson_drm_device *ldev) +{ + int i, ret; + struct drm_encoder *encoder; + struct drm_connector *connector; + + ldev->mode_info[0].mode_config_initialized = true; + ldev->mode_info[1].mode_config_initialized = true; + + ldev->dev->mode_config.max_width = LOONGSON_MAX_FB_WIDTH; + ldev->dev->mode_config.max_height = LOONGSON_MAX_FB_HEIGHT; + + ldev->dev->mode_config.cursor_width = 32; + ldev->dev->mode_config.cursor_height = 32; + + ldev->dev->mode_config.allow_fb_modifiers = true; + + ret = loongson_i2c_init(ldev); + if (ret < 0) { + DRM_ERROR("Failed to initialize i2c\n"); + return ret; + } + + loongson_crtc_init(ldev); + + for (i=0; inum_crtc; i++) { + DRM_DEBUG("loongson drm encoder init\n"); + ldev->mode_info[i].crtc = &ldev->lcrtc[i]; + encoder = loongson_encoder_init(ldev->dev, i); + if (!encoder) { + DRM_ERROR("loongson_encoder_init failed\n"); + return -1; + } + ldev->mode_info[i].encoder = to_loongson_encoder(encoder); + + DRM_DEBUG("loongson drm connector init\n"); + connector = loongson_connector_init(ldev->dev, i); + if (!connector) { + DRM_ERROR("loongson_connector_init failed\n"); + return -1; + } + ldev->mode_info[i].connector = to_loongson_connector(connector); + + drm_connector_attach_encoder(connector, encoder); + if (poll_connector) + connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; + } + + return 0; +} + +/** + * loongson_modeset_fini --- deinit kernel mode setting + * + * @ldev: pointer to loongson_drm_device structure + * + * RETURN + */ +void loongson_modeset_fini(struct loongson_drm_device *ldev) +{ +} + +static int loongson_detect_chip(struct loongson_drm_device *ldev) +{ + struct pci_dev *pdev; + + pdev = pci_get_device(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, NULL); + if (pdev) { + ldev->chip = dc_7a1000; + ldev->gpu_pdev = pci_get_device(PCI_VENDOR_ID_LOONGSON, 0x7a15, NULL); + DRM_INFO("Set LS7A1000 DC device\n"); + return 0; + } + + pdev = pci_get_device(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, NULL); + if (pdev) { + ldev->chip = dc_7a2000; + ldev->gpu_pdev = pci_get_device(PCI_VENDOR_ID_LOONGSON, 0x7a25, NULL); + DRM_INFO("Set LS7A2000 DC device\n"); + return 0; + } + + return -1; +} + +/** + * loongson_vga_load - setup chip and create an initial config + * @dev: DRM device + * @flags: startup flags + * + * The driver load routine has to do several things: + * - initialize the memory manager + * - allocate initial config memory + * - setup the DRM framebuffer with the allocated memory + */ +static int loongson_drm_load(struct drm_device *dev, unsigned long flags) +{ + int r, ret, irq; + struct loongson_drm_device *ldev; + + dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)); + + ldev = devm_kzalloc(dev->dev, sizeof(struct loongson_drm_device), GFP_KERNEL); + if (ldev == NULL) + return -ENOMEM; + dev->dev_private = (void *)ldev; + ldev->dev = dev; + + ret = loongson_detect_chip(ldev); + if (ret) + dev_err(dev->dev, "Fatal error during detect chip: %d\n", ret); + + ret = loongson_drm_device_init(dev, flags); + DRM_DEBUG("end loongson drm device init.\n"); + + drm_mode_config_init(dev); + dev->mode_config.funcs = (void *)&loongson_mode_funcs; + dev->mode_config.preferred_depth = 24; + dev->mode_config.prefer_shadow = 1; + + irq = dev->pdev->irq; + dev_set_drvdata(dev->dev, dev); + pci_set_drvdata(dev->pdev, dev); + + r = loongson_modeset_init(ldev); + if (r) + dev_err(dev->dev, "Fatal error during modeset init: %d\n", r); + + r = drm_irq_install(dev, irq); + if (r) + dev_err(dev->dev, "Fatal error during irq install: %d\n", r); + + ldev->inited = true; + drm_mode_config_reset(dev); + + r = drm_vblank_init(dev, ldev->num_crtc); + if (r) + dev_err(dev->dev, "Fatal error during vblank init: %d\n", r); + + /* Make small buffers to store a hardware cursor (double buffered icon updates) */ + ldev->cursor = drm_gem_cma_create(dev, roundup(32*32*4, PAGE_SIZE)); + + drm_kms_helper_poll_init(dev); + + drm_fb_helper_remove_conflicting_framebuffers(NULL, "loongson-drmfb", false); + + return 0; +} + +/** + * loongson_drm_unload--release drm resource + * + * @dev: pointer to drm_device + * + */ +static void loongson_drm_unload(struct drm_device *dev) +{ + struct loongson_drm_device *ldev = dev->dev_private; + + if (ldev == NULL) + return; + + loongson_modeset_fini(ldev); + drm_mode_config_cleanup(dev); + dev->dev_private = NULL; + dev_set_drvdata(dev->dev, NULL); + ldev->inited = false; + + return; +} + +/** + * loongson_drm_open -Driver callback when a new struct drm_file is opened. + * Useful for setting up driver-private data structures like buffer allocators, + * execution contexts or similar things. + * + * @dev DRM device + * @file DRM file private date + * + * RETURN + * 0 on success, a negative error code on failure, which will be promoted to + * userspace as the result of the open() system call. + */ +static int loongson_drm_open(struct drm_device *dev, struct drm_file *file) +{ + file->driver_priv = NULL; + + DRM_DEBUG("open: dev=%p, file=%p", dev, file); + + return 0; +} + +DEFINE_DRM_GEM_CMA_FOPS(fops); + +/** + * loongson_drm_driver - DRM device structure + * + * .load: driver callback to complete initialization steps after the driver is registered + * .unload:Reverse the effects of the driver load callback + * .open:Driver callback when a new struct drm_file is opened + * .fops:File operations for the DRM device node. + * .gem_free_object:deconstructor for drm_gem_objects + * .dumb_create:This creates a new dumb buffer in the driver’s backing storage manager + * (GEM, TTM or something else entirely) and returns the resulting buffer handle. + * This handle can then be wrapped up into a framebuffer modeset object + * .dumb_map_offset:Allocate an offset in the drm device node’s address space + * to be able to memory map a dumb buffer + * .dump_destory:This destroys the userspace handle for the given dumb backing storage buffer + */ +static struct drm_driver loongson_drm_driver = { + .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_HAVE_IRQ | DRIVER_ATOMIC, + .open = loongson_drm_open, + .fops = &fops, + + .dumb_create = drm_gem_cma_dumb_create, + .gem_free_object_unlocked = drm_gem_cma_free_object, + .gem_vm_ops = &drm_gem_cma_vm_ops, + + .prime_handle_to_fd = drm_gem_prime_handle_to_fd, + .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + + .gem_prime_import = drm_gem_prime_import, + .gem_prime_export = drm_gem_prime_export, + + .gem_prime_get_sg_table = drm_gem_cma_prime_get_sg_table, + .gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table, + .gem_prime_vmap = drm_gem_cma_prime_vmap, + .gem_prime_vunmap = drm_gem_cma_prime_vunmap, + .gem_prime_mmap = drm_gem_cma_prime_mmap, + + .irq_handler = loongson_irq_handler, + .irq_preinstall = loongson_irq_preinstall, + .irq_postinstall = loongson_irq_postinstall, + .irq_uninstall = loongson_irq_uninstall, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +/** + * loongson_drm_pci_devices -- pci device id info + * + * __u32 vendor, device Vendor and device ID or PCI_ANY_ID + * __u32 subvendor, subdevice Subsystem ID's or PCI_ANY_ID + * __u32 class, class_mask (class,subclass,prog-if) triplet + * kernel_ulong_t driver_data Data private to the driver + */ +static struct pci_device_id loongson_drm_pci_devices[] = { + {PCI_DEVICE(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1)}, + {PCI_DEVICE(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2)}, + {0, 0, 0, 0, 0, 0, 0} +}; + +MODULE_DEVICE_TABLE(pci, loongson_drm_pci_devices); + +/** + * loongson_drm_pci_register -- add pci device + * + * @pdev PCI device + * @ent pci device id + */ +static int loongson_drm_pci_register(struct pci_dev *pdev, + const struct pci_device_id *ent) + +{ + int ret; + struct drm_device *dev; + + dev = drm_dev_alloc(&loongson_drm_driver, &pdev->dev); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + ret = pci_enable_device(pdev); + if (ret) + goto err_free; + + dev->pdev = pdev; + + loongson_drm_load(dev, 0x0); + + ret = drm_dev_register(dev, 0); + if (ret) + goto err_pdev; + + drm_fbdev_generic_setup(dev, 32); + + return 0; + +err_pdev: + pci_disable_device(pdev); +err_free: + drm_dev_put(dev); + return ret; +} + +/** + * loongson_drm_pci_unregister -- release drm device + * + * @pdev PCI device + */ +static void loongson_drm_pci_unregister(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + loongson_drm_unload(dev); + drm_dev_put(dev); +} + +/* + * Suspend & resume. + */ +/* + * loongson_drm_suspend - initiate device suspend + * + * @pdev: drm dev pointer + * @state: suspend state + * + * Puts the hw in the suspend state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver suspend. + */ +int loongson_drm_suspend(struct drm_device *dev, bool suspend) +{ + struct loongson_drm_device *ldev; + + if (dev == NULL || dev->dev_private == NULL) + return -ENODEV; + + ldev = dev->dev_private; + + drm_kms_helper_poll_disable(dev); + ldev->state = drm_atomic_helper_suspend(dev); + + pci_save_state(dev->pdev); + if (suspend) { + /* Shut down the device */ + pci_disable_device(dev->pdev); + pci_set_power_state(dev->pdev, PCI_D3hot); + } + + console_lock(); + drm_fb_helper_set_suspend(ldev->dev->fb_helper, 1); + console_unlock(); + + return 0; +} + +/* + * * loongson_drm_resume - initiate device suspend + * + * @pdev: drm dev pointer + * @state: suspend state + * + * Puts the hw in the suspend state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver suspend. + */ + +int loongson_drm_resume(struct drm_device *dev, bool resume) +{ + struct loongson_drm_device *ldev = dev->dev_private; + + console_lock(); + + if (resume) { + pci_set_power_state(dev->pdev, PCI_D0); + pci_restore_state(dev->pdev); + if (pci_enable_device(dev->pdev)) { + console_unlock(); + return -1; + } + } + + /* blat the mode back in */ + drm_atomic_helper_resume(dev, ldev->state); + + drm_kms_helper_poll_enable(dev); + + drm_fb_helper_set_suspend(ldev->dev->fb_helper, 0); + + console_unlock(); + + return 0; +} + +/** + * loongson_drm_pm_suspend + * + * @dev pointer to the device + * + * Executed before putting the system into a sleep state in which the + * contents of main memory are preserved. + */ +static int loongson_drm_pm_suspend(struct device *dev) +{ + struct drm_device *drm_dev = dev_get_drvdata(dev); + + return loongson_drm_suspend(drm_dev, true); +} + +/** + * loongson_drm_pm_resume + * + * @dev pointer to the device + * + * Executed after waking the system up from a sleep state in which the + * contents of main memory were preserved. + */ +static int loongson_drm_pm_resume(struct device *dev) +{ + struct drm_device *drm_dev = dev_get_drvdata(dev); + + return loongson_drm_resume(drm_dev, true); +} + +/** + * loongson_drm_pm_freeze + * + * @dev pointer to device + * + * Hibernation-specific, executed before creating a hibernation image. + * Analogous to @suspend(), but it should not enable the device to signal + * wakeup events or change its power state. + */ +static int loongson_drm_pm_freeze(struct device *dev) +{ + struct drm_device *drm_dev = dev_get_drvdata(dev); + + return loongson_drm_suspend(drm_dev, false); +} + +/** + * loongson_drm_pm_draw + * + * @dev pointer to device + * + * Hibernation-specific, executed after creating a hibernation image OR + * if the creation of an image has failed. Also executed after a failing + * attempt to restore the contents of main memory from such an image. + * Undo the changes made by the preceding @freeze(), so the device can be + * operated in the same way as immediately before the call to @freeze(). + */ +static int loongson_drm_pm_thaw(struct device *dev) +{ + struct drm_device *drm_dev = dev_get_drvdata(dev); + + return loongson_drm_resume(drm_dev, false); +} + +#define loongson_drm_pm_poweroff loongson_drm_pm_freeze +#define loongson_drm_pm_restore loongson_drm_pm_resume + +/* + * * struct dev_pm_ops - device PM callbacks + * + *@suspend: Executed before putting the system into a sleep state in which the + * contents of main memory are preserved. + *@resume: Executed after waking the system up from a sleep state in which the + * contents of main memory were preserved. + *@freeze: Hibernation-specific, executed before creating a hibernation image. + * Analogous to @suspend(), but it should not enable the device to signal + * wakeup events or change its power state. The majority of subsystems + * (with the notable exception of the PCI bus type) expect the driver-level + * @freeze() to save the device settings in memory to be used by @restore() + * during the subsequent resume from hibernation. + *@thaw: Hibernation-specific, executed after creating a hibernation image OR + * if the creation of an image has failed. Also executed after a failing + * attempt to restore the contents of main memory from such an image. + * Undo the changes made by the preceding @freeze(), so the device can be + * operated in the same way as immediately before the call to @freeze(). + *@poweroff: Hibernation-specific, executed after saving a hibernation image. + * Analogous to @suspend(), but it need not save the device's settings in + * memory. + *@restore: Hibernation-specific, executed after restoring the contents of main + * memory from a hibernation image, analogous to @resume(). + */ +static const struct dev_pm_ops loongson_drm_pm_ops = { + .suspend = loongson_drm_pm_suspend, + .resume = loongson_drm_pm_resume, + .freeze = loongson_drm_pm_freeze, + .thaw = loongson_drm_pm_thaw, + .poweroff = loongson_drm_pm_poweroff, + .restore = loongson_drm_pm_restore, +}; + +/** + * loongson_drm_pci_driver -- pci driver structure + * + * .id_table : must be non-NULL for probe to be called + * .probe: New device inserted + * .remove: Device removed + * .resume: Device suspended + * .suspend: Device woken up + */ +static struct pci_driver loongson_drm_pci_driver = { + .name = DRIVER_NAME, + .id_table = loongson_drm_pci_devices, + .probe = loongson_drm_pci_register, + .remove = loongson_drm_pci_unregister, + .driver.pm = &loongson_drm_pm_ops, +}; + +/** + * loongson_drm_pci_init() -- kernel module init function + */ +static int __init loongson_drm_init(void) +{ + int ret; + struct pci_dev *pdev = NULL; + + /* If external graphics card exist, use it as default */ + while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev))) { + if (pdev->vendor == PCI_VENDOR_ID_ATI) + return 0; + if (pdev->vendor == 0x1a03) /* ASpeed */ + return 0; + } + + ret = pci_register_driver(&loongson_drm_pci_driver); + + return ret; +} + +/** + * loongson_drm_pci_exit() -- kernel module exit function + */ +static void __exit loongson_drm_exit(void) +{ + pci_unregister_driver(&loongson_drm_pci_driver); +} + +module_init(loongson_drm_init); +module_exit(loongson_drm_exit); + +MODULE_AUTHOR("Chen Zhu "); +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("Loongson LS7A DRM Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/loongson/loongson_drv.h b/drivers/gpu/drm/loongson/loongson_drv.h new file mode 100644 index 0000000000000000000000000000000000000000..dbe3ebf951cd5ff10cc50b23d7c9e37afae8693d --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_drv.h @@ -0,0 +1,241 @@ +#ifndef __LOONGSON_DRV_H__ +#define __LOONGSON_DRV_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include "loongson_i2c.h" +#include "loongson_vbios.h" + +#define to_loongson_crtc(x) container_of(x, struct loongson_crtc, base) +#define to_loongson_encoder(x) container_of(x, struct loongson_encoder, base) +#define to_loongson_connector(x) container_of(x, struct loongson_connector, base) + +#define LOONGSON_MAX_FB_HEIGHT 4096 +#define LOONGSON_MAX_FB_WIDTH 4096 + +#define CUR_WIDTH_SIZE 32 +#define CUR_HEIGHT_SIZE 32 + +#define LO_OFF 0 +#define HI_OFF 8 + +#define LS7A_PIX0_PLL (void *)TO_UNCACHE(LS7A_CHIPCFG_REG_BASE + 0x04b0) +#define LS7A_PIX1_PLL (void *)TO_UNCACHE(LS7A_CHIPCFG_REG_BASE + 0x04c0) + +#define CURIOSET_CORLOR 0x4607 +#define CURIOSET_POSITION 0x4608 +#define CURIOLOAD_ARGB 0x4609 +#define CURIOLOAD_IMAGE 0x460A +#define CURIOHIDE_SHOW 0x460B +#define FBEDID_GET 0X860C + +#define CRTC_REG_OFFSET 0x10 + +#define CFG_FMT GENMASK(2,0) +#define CFG_FBSWITCH BIT(7) +#define CFG_ENABLE BIT(8) +#define CFG_PANELSWITCH BIT(9) +#define CFG_FBNUM_BIT 11 +#define CFG_FBNUM BIT(11) +#define CFG_GAMMAR BIT(12) +#define CFG_RESET BIT(20) + +#define FB_CFG_DVO_REG (0x1240) +#define FB_ADDR0_DVO_REG (0x1260) +#define FB_ADDR1_DVO_REG (0x1580) +#define FB_STRI_DVO_REG (0x1280) +#define FB_DITCFG_DVO_REG (0x1360) +#define FB_DITTAB_LO_DVO_REG (0x1380) +#define FB_DITTAB_HI_DVO_REG (0x13a0) +#define FB_PANCFG_DVO_REG (0x13c0) +#define FB_PANTIM_DVO_REG (0x13e0) +#define FB_HDISPLAY_DVO_REG (0x1400) +#define FB_HSYNC_DVO_REG (0x1420) +#define FB_VDISPLAY_DVO_REG (0x1480) +#define FB_VSYNC_DVO_REG (0x14a0) +#define FB_GAMINDEX_DVO_REG (0x14e0) +#define FB_GAMDATA_DVO_REG (0x1500) + +#define HDMI_ZONEIDLE_REG (0x1700) +#define HDMI_CTRL_REG (0x1720) + +#define HDMI_AUDIO_BUF_REG (0x1740) +#define HDMI_AUDIO_NCFG_REG (0x1760) +#define HDMI_AUDIO_CTSCFG_REG (0x1780) +#define HDMI_AUDIO_CTSCALCFG_REG (0x17a0) +#define HDMI_AUDIO_INFOFRAME_REG (0x17c0) +#define HDMI_AUDIO_SAMPLE_REG (0x17e0) + +#define HDMI_PHY_CTRL_REG (0x1800) +#define HDMI_PHY_PLLCFG_REG (0x1820) + +#define FB_CUR_CFG_REG (0x1520) +#define FB_CUR_ADDR_REG (0x1530) +#define FB_CUR_LOC_ADDR_REG (0x1540) +#define FB_CUR_BACK_REG (0x1550) +#define FB_CUR_FORE_REG (0x1560) +#define FB_INT_REG (0x1570) + +#define INT_DVO1_VSYNC 0 +#define INT_DVO1_HSYNC 1 +#define INT_DVO0_VSYNC 2 +#define INT_DVO0_HSYNC 3 +#define INT_CURSOR_FB_END 4 +#define INT_DVO1_FB_END 5 +#define INT_DVO0_FB_END 6 + +#define MAX_CRTC 2 + +enum loongson_chip { + dc_7a1000, + dc_7a2000 +}; + +struct pix_pll { + unsigned int l2_div; + unsigned int l1_loopc; + unsigned int l1_frefc; +}; + +struct config_reg { + u8 dev_addr; + u8 reg; + u8 value; +} __packed; + +struct cfg_encoder { + u8 reg_num; + u32 hdisplay; + u32 vdisplay; + struct config_reg config_regs[256]; +}; + +struct loongson_crtc { + struct drm_crtc base; + unsigned int crtc_id; + uint32_t cfg_reg; + struct drm_plane *primary; /* Primary panel belongs to this crtc */ + struct drm_pending_vblank_event *event; + struct loongson_drm_device *ldev; +}; + +struct loongson_encoder { + struct drm_encoder base; + u32 type; + u32 i2c_id; + int encoder_id; + struct loongson_i2c *i2c; + struct loongson_crtc *lcrtc; /* Binding crtc, not actual one */ +}; + +struct loongson_connector { + struct drm_connector base; + u16 id; + u32 type; + u16 i2c_id; + u16 hotplug; + u16 edid_method; + u8 *vbios_edid; + struct loongson_i2c *i2c; + struct loongson_drm_device *ldev; +}; + +struct loongson_mode_info { + bool mode_config_initialized; + struct loongson_crtc *crtc; + struct loongson_encoder *encoder; + struct loongson_connector *connector; +}; + +struct loongson_drm_device { + struct drm_device *dev; + struct drm_atomic_state *state; + struct pci_dev *gpu_pdev; /* LS7A gpu device info */ + + resource_size_t mmio_base; + resource_size_t mmio_size; + void __iomem *mmio; + uint32_t int_reg; + + struct drm_display_mode mode; + struct loongson_mode_info mode_info[2]; + struct drm_gem_cma_object *cursor; + + int num_crtc; + struct loongson_crtc lcrtc[MAX_CRTC]; + struct loongson_i2c i2c_bus[DC_I2C_BUS_MAX]; + + void *vbios; + struct list_head desc_list; + + bool inited; + bool suspended; + bool cursor_showed; + int cursor_crtc_id; + + int connector_active0; + int connector_active1; + + enum loongson_chip chip; +}; + +extern spinlock_t loongson_reglock; + +/* FIXME: LS7A2000's switch_panel is not available, just return false. */ +static inline bool clone_mode(struct loongson_drm_device *ldev) +{ + return false; + + if (ldev->num_crtc < 2) + return true; + if (ldev->mode_info[0].connector->base.status != connector_status_connected) + return true; + if (ldev->mode_info[1].connector->base.status != connector_status_connected) + return true; + if (ldev->lcrtc[0].base.x || ldev->lcrtc[0].base.y) + return false; + if (ldev->lcrtc[1].base.x || ldev->lcrtc[1].base.y) + return false; + + return true; +} + +int loongson_irq_enable_vblank(struct drm_device *dev,unsigned int crtc_id); +void loongson_irq_disable_vblank(struct drm_device *dev,unsigned int crtc_id); +irqreturn_t loongson_irq_handler(int irq,void *arg); +void loongson_irq_preinstall(struct drm_device *dev); +int loongson_irq_postinstall(struct drm_device *dev); +void loongson_irq_uninstall(struct drm_device *dev); + +u32 crtc_read(struct loongson_crtc *lcrtc, u32 offset); +void crtc_write(struct loongson_crtc *lcrtc, u32 offset, u32 val); + +int loongson_gpio_init(struct loongson_drm_device *ldev); +int loongson_crtc_init(struct loongson_drm_device *ldev); +struct drm_encoder *loongson_encoder_init(struct drm_device *dev, unsigned int index); +struct drm_connector *loongson_connector_init(struct drm_device *dev, unsigned int index); + +int loongson_fbdev_init(struct loongson_drm_device *ldev); +void loongson_fbdev_fini(struct loongson_drm_device *ldev); +void loongson_fbdev_restore_mode(struct loongson_drm_device *ldev); + +int loongson_drm_drm_suspend(struct drm_device *dev, bool suspend, + bool fbcon, bool freeze); +int loongson_drm_drm_resume(struct drm_device *dev, bool resume, bool fbcon); + /* loongson_cursor.c */ +int loongson_crtc_cursor_set2(struct drm_crtc *crtc, struct drm_file *file_priv, + uint32_t handle, uint32_t width, uint32_t height, int32_t hot_x, int32_t hot_y); +int loongson_crtc_cursor_move(struct drm_crtc *crtc, int x, int y); + +#endif diff --git a/drivers/gpu/drm/loongson/loongson_encoder.c b/drivers/gpu/drm/loongson/loongson_encoder.c new file mode 100644 index 0000000000000000000000000000000000000000..f2d4c49f5319252b01e26491e2c91a3100768181 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_encoder.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Copyright (C) 2019 Lemote Inc. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * Jiaxun Yang + */ + +#include +#include "loongson_drv.h" + +/** + * loongson_encoder_destroy + * + * @encoder: encoder object + * + * Clean up encoder resources + */ +static void loongson_encoder_destroy(struct drm_encoder *encoder) +{ + struct loongson_encoder *loongson_encoder = to_loongson_encoder(encoder); + drm_encoder_cleanup(encoder); + kfree(loongson_encoder); +} + +static int loongson_encoder_atomic_check(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + return 0; +} + +static void loongson_encoder_atomic_mode_set(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + unsigned long flags; + struct loongson_encoder *lenc = to_loongson_encoder(encoder); + struct loongson_crtc *lcrtc_origin = lenc->lcrtc; + struct loongson_crtc *lcrtc_current = to_loongson_crtc(crtc_state->crtc); + + if (lcrtc_origin->crtc_id != lcrtc_current->crtc_id) + lcrtc_origin->cfg_reg |= CFG_PANELSWITCH; + else + lcrtc_origin->cfg_reg &= ~CFG_PANELSWITCH; + + spin_lock_irqsave(&loongson_reglock, flags); + crtc_write(lcrtc_origin, FB_CFG_DVO_REG, lcrtc_origin->cfg_reg); + spin_unlock_irqrestore(&loongson_reglock, flags); +} + +/** + * These provide the minimum set of functions required to handle a encoder + * + * Helper operations for encoders + */ +static const struct drm_encoder_helper_funcs loongson_encoder_helper_funcs = { + .atomic_check = loongson_encoder_atomic_check, + .atomic_mode_set = loongson_encoder_atomic_mode_set, +}; + +/** + * These provide the minimum set of functions required to handle a encoder + * + * Encoder controls,encoder sit between CRTCs and connectors + */ +static const struct drm_encoder_funcs loongson_encoder_encoder_funcs = { + .destroy = loongson_encoder_destroy, +}; + +static void loongson_hdmi_init(struct loongson_drm_device *ldev, int index) +{ + u32 val; + int offset = index * 0x10; + volatile void __iomem *base = ldev->mmio; + + spin_lock(&loongson_reglock); + writel(0x287, base + HDMI_CTRL_REG + offset); + + writel(0x00400040, base + HDMI_ZONEIDLE_REG + offset); + + writel(6272, base + HDMI_AUDIO_NCFG_REG + offset); + writel(0x80000000, base + HDMI_AUDIO_CTSCFG_REG + offset); + + writel(0x11, base + HDMI_AUDIO_INFOFRAME_REG + offset); + val = readl(base + HDMI_AUDIO_INFOFRAME_REG + offset) | 0x4; + writel(val, base + HDMI_AUDIO_INFOFRAME_REG + offset); + + writel(0x1, base + HDMI_AUDIO_SAMPLE_REG + offset); + spin_unlock(&loongson_reglock); + + DRM_DEBUG("Loongson HDMI init finish.\n"); +} + +/** + * loongson_encoder_init + * + * @dev: point to the drm_device structure + * + * Init encoder + */ +struct drm_encoder *loongson_encoder_init(struct drm_device *dev, unsigned int index) +{ + struct drm_encoder *encoder; + struct loongson_encoder *loongson_encoder; + struct loongson_drm_device *ldev = dev->dev_private; + + loongson_encoder = kzalloc(sizeof(struct loongson_encoder), GFP_KERNEL); + if (!loongson_encoder) + return NULL; + + loongson_encoder->encoder_id = index; + loongson_encoder->i2c = &ldev->i2c_bus[index]; + loongson_encoder->lcrtc = &ldev->lcrtc[index]; + loongson_encoder->type = get_encoder_type(ldev, index); + encoder = &loongson_encoder->base; + + if (loongson_encoder->type == DRM_MODE_ENCODER_TMDS) + loongson_hdmi_init(ldev, index); + + encoder->possible_crtcs = BIT(index); + encoder->possible_clones = BIT(1) | BIT(0); + /* encoder->possible_crtcs = BIT(1) | BIT(0); */ + + drm_encoder_helper_add(encoder, &loongson_encoder_helper_funcs); + drm_encoder_init(dev, encoder, &loongson_encoder_encoder_funcs, + loongson_encoder->type, NULL); + + return encoder; +} diff --git a/drivers/gpu/drm/loongson/loongson_i2c.c b/drivers/gpu/drm/loongson/loongson_i2c.c new file mode 100644 index 0000000000000000000000000000000000000000..55a59ccfd78ebec5cdfd61db8b6aa97272948230 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_i2c.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "loongson_i2c.h" +#include "loongson_drv.h" +#include "loongson_vbios.h" + +u32 ls7a_mm_rreg(struct loongson_drm_device *ldev, u32 offset) +{ + return readl(ldev->mmio + offset); +} + +void ls7a_mm_wreg(struct loongson_drm_device *ldev, u32 offset, u32 val) +{ + writel(val, ldev->mmio + offset); +} + +static inline void __dc_gpio_set_dir(struct loongson_drm_device *ldev, + unsigned int pin, int input) +{ + u32 temp; + + temp = ls7a_mm_rreg(ldev, LS7A_DC_GPIO_CFG_OFFSET); + if (input) + temp |= 1UL << pin; + else + temp &= ~(1UL << pin); + ls7a_mm_wreg(ldev, LS7A_DC_GPIO_CFG_OFFSET, temp); +} + +static void __dc_gpio_set_val(struct loongson_drm_device *ldev, unsigned int pin, + int high) +{ + u32 temp; + + temp = ls7a_mm_rreg(ldev, LS7A_DC_GPIO_OUT_OFFSET); + if (high) + temp |= 1UL << pin; + else + temp &= ~(1UL << pin); + ls7a_mm_wreg(ldev, LS7A_DC_GPIO_OUT_OFFSET, temp); +} + +static void loongson_i2c_set_data(void *i2c, int value) +{ + struct loongson_i2c *li2c = i2c; + struct loongson_drm_device *ldev = li2c->ldev; + unsigned int pin = li2c->data; + + if (value) + __dc_gpio_set_dir(ldev, pin, 1); + else { + __dc_gpio_set_val(ldev, pin, 0); + __dc_gpio_set_dir(ldev, pin, 0); + } +} + +static void loongson_i2c_set_clock(void *i2c, int value) +{ + struct loongson_i2c *li2c = i2c; + struct loongson_drm_device *ldev = li2c->ldev; + unsigned int pin = li2c->clock; + + if (value) + __dc_gpio_set_dir(ldev, pin, 1); + else { + __dc_gpio_set_val(ldev, pin, 0); + __dc_gpio_set_dir(ldev, pin, 0); + } +} + +static int loongson_i2c_get_data(void *i2c) +{ + int val; + struct loongson_i2c *li2c = i2c; + struct loongson_drm_device *ldev = li2c->ldev; + unsigned int pin = li2c->data; + + val = ls7a_mm_rreg(ldev, LS7A_DC_GPIO_IN_OFFSET); + + return (val >> pin) & 1; +} + +static int loongson_i2c_get_clock(void *i2c) +{ + int val; + struct loongson_i2c *li2c = i2c; + struct loongson_drm_device *ldev = li2c->ldev; + unsigned int pin = li2c->clock; + + val = ls7a_mm_rreg(ldev, LS7A_DC_GPIO_IN_OFFSET); + + return (val >> pin) & 1; +} + +static int loongson_i2c_create(struct loongson_drm_device *ldev, + struct loongson_i2c *li2c, const char *name) +{ + int ret; + unsigned int i2c_num; + struct i2c_adapter *i2c_adapter; + struct i2c_algo_bit_data *i2c_algo_data; + + i2c_num = li2c->i2c_id; + i2c_adapter = kzalloc(sizeof(struct i2c_adapter), GFP_KERNEL); + if (!i2c_adapter) + return -ENOMEM; + + i2c_algo_data = kzalloc(sizeof(struct i2c_algo_bit_data), GFP_KERNEL); + if (!i2c_algo_data) { + ret = -ENOMEM; + goto free_adapter; + } + + i2c_adapter->owner = THIS_MODULE; + i2c_adapter->class = I2C_CLASS_DDC; + i2c_adapter->algo_data = i2c_algo_data; + i2c_adapter->dev.parent = ldev->dev->dev; + i2c_adapter->nr = -1; + snprintf(i2c_adapter->name, sizeof(i2c_adapter->name), "%s%d", name, + i2c_num); + + li2c->data = i2c_num * 2; + li2c->clock = i2c_num * 2 + 1; + DRM_INFO("Created dc-i2c%d, sda=%d, scl=%d\n", i2c_num, li2c->data, + li2c->clock); + + i2c_algo_data->setsda = loongson_i2c_set_data; + i2c_algo_data->setscl = loongson_i2c_set_clock; + i2c_algo_data->getsda = loongson_i2c_get_data; + i2c_algo_data->getscl = loongson_i2c_get_clock; + i2c_algo_data->udelay = DC_I2C_TON; + i2c_algo_data->timeout = usecs_to_jiffies(2200); /* from VESA */ + + ret = i2c_bit_add_numbered_bus(i2c_adapter); + if (ret) + goto free_algo_data; + + li2c->adapter = i2c_adapter; + i2c_algo_data->data = li2c; + i2c_set_adapdata(li2c->adapter, li2c); + li2c->init = true; + li2c->ldev = ldev; + DRM_INFO("Register i2c algo-bit adapter [%s]\n", i2c_adapter->name); + + return 0; + +free_algo_data: + DRM_ERROR("Failed to register i2c adapter %s\n", i2c_adapter->name); + kfree(i2c_algo_data); +free_adapter: + kfree(i2c_adapter); + + return ret; +} + +int loongson_gpio_init(struct loongson_drm_device *ldev) +{ + int pin; + + /* set gpio dir output 0-3 */ + for (pin = 0; pin < 4; pin++) { + __dc_gpio_set_val(ldev, pin, 0); + __dc_gpio_set_dir(ldev, pin, 0); + } + + return 0; +} + +int loongson_i2c_init(struct loongson_drm_device *ldev) +{ + int ret, i; + + ret = get_loongson_i2c(ldev); + if (ret != true) { + DRM_ERROR("Failed to get i2c_id form vbios\n"); + return -ENODEV; + } + + for (i = 0; i < DC_I2C_BUS_MAX; i++) { + if (!ldev->i2c_bus[i].use) + continue; + ldev->i2c_bus[i].i2c_id = i; + ret = loongson_i2c_create(ldev, &ldev->i2c_bus[i], DC_I2C_NAME); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/loongson/loongson_i2c.h b/drivers/gpu/drm/loongson/loongson_i2c.h new file mode 100644 index 0000000000000000000000000000000000000000..97d3d139ca73102d942494d21a76023438c185fa --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_i2c.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2020 Loongson Technology Co., Ltd. + * Authors: + * sunhao + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#ifndef __LOONGSON_I2C_H__ +#define __LOONGSON_I2C_H__ + +#include +#include +#include + +/* Modify this marco to config i2c bus speed, bus_freq = 500 / T */ +/* Eg: i2c_bus_freq=100k when T=5 */ +#define DC_I2C_TON 5 +#define DC_I2C_NAME "ls7a_dc_i2c" +#define DC_I2C_BUS_MAX 2 + +/* Loongson 7A display controller proprietary GPIOs */ +#define LS7A_DC_GPIO_CFG_OFFSET (0x1660) +#define LS7A_DC_GPIO_IN_OFFSET (0x1650) +#define LS7A_DC_GPIO_OUT_OFFSET (0x1650) + +struct loongson_drm_device; + +struct loongson_i2c { + struct loongson_drm_device *ldev; + struct i2c_client *ddc_client; + struct i2c_adapter *adapter; + u32 data, clock; + bool use, init; + u32 i2c_id; +}; + +int loongson_i2c_init(struct loongson_drm_device *ldev); + +#endif diff --git a/drivers/gpu/drm/loongson/loongson_irq.c b/drivers/gpu/drm/loongson/loongson_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..aeed283a1a54fe8b4af19a7a47401e903abd2018 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_irq.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include "loongson_drv.h" + +irqreturn_t loongson_irq_handler(int irq, void *arg) +{ + unsigned int val; + struct drm_device *dev = (struct drm_device *) arg; + struct loongson_drm_device *ldev = dev->dev_private; + volatile void __iomem *base = ldev->mmio; + + val = readl(base + FB_INT_REG); + spin_lock(&loongson_reglock); + writel(val, base + FB_INT_REG); + spin_unlock(&loongson_reglock); + + if (val & BIT(INT_DVO0_FB_END)){ + drm_crtc_handle_vblank(&ldev->lcrtc[0].base); + } + + if (val & BIT(INT_DVO1_FB_END)){ + drm_crtc_handle_vblank(&ldev->lcrtc[1].base); + } + + spin_lock(&loongson_reglock); + writel(ldev->int_reg, base + FB_INT_REG); + spin_unlock(&loongson_reglock); + + return IRQ_HANDLED; +} + +void loongson_irq_preinstall(struct drm_device *dev) +{ + unsigned long flags; + struct loongson_drm_device *ldev = dev->dev_private; + volatile void __iomem *base = ldev->mmio; + + /* disable interupt */ + spin_lock_irqsave(&loongson_reglock, flags); + writel(0x0000 << 16, base + FB_INT_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); +} + +int loongson_irq_postinstall(struct drm_device *dev) +{ + return 0; +} + +void loongson_irq_uninstall(struct drm_device *dev) +{ + unsigned long flags; + struct loongson_drm_device *ldev = dev->dev_private; + volatile void __iomem *base = ldev->mmio; + + /* disable interupt */ + spin_lock_irqsave(&loongson_reglock, flags); + writel(0x0000 << 16, base + FB_INT_REG); + spin_unlock_irqrestore(&loongson_reglock, flags); +} diff --git a/drivers/gpu/drm/loongson/loongson_vbios.c b/drivers/gpu/drm/loongson/loongson_vbios.c new file mode 100644 index 0000000000000000000000000000000000000000..38bf164f7b16b680cefebca911b327c9282ba906 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_vbios.c @@ -0,0 +1,892 @@ +/* + * Copyright (c) 2018 Loongson Technology Co., Ltd. + * Authors: + * Chen Zhu + * Yaling Fang + * Dandan Zhang + * Huacai Chen + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include "loongson_drv.h" +#include "loongson_vbios.h" + +#define VBIOS_START 0x1000 +#define VBIOS_SIZE 0x40000 +#define VBIOS_OFFSET 0x100000 +#define VBIOS_DESC_OFFSET 0x6000 +#define VBIOS_TITLE "Loongson-VBIOS" + +/* VBIOS INFO ADDRESS TABLE */ +struct acpi_viat_table { + struct acpi_table_header header; + unsigned long vbios_addr; +} __packed; + +static u32 get_vbios_version(struct loongson_vbios *vbios) +{ + u32 minor, major, version; + + minor = vbios->version_minor; + major = vbios->version_major; + version = major * 10 + minor; + + return version; +} + +static bool parse_vbios_i2c(struct desc_node *this, struct vbios_cmd *cmd) +{ + bool ret = true; + int i, num, size; + struct vbios_i2c *i2c; + struct vbios_i2c *vbios_i2c = NULL; + struct loongson_i2c *val = (struct loongson_i2c *)cmd->res; + + size = this->desc->size; + vbios_i2c = kzalloc(size, GFP_KERNEL); + if (!vbios_i2c) + return false; + + memset(vbios_i2c, 0xff, size); + memcpy(vbios_i2c, this->data, size); + num = size / sizeof(*vbios_i2c); + + i2c = vbios_i2c; + for (i = 0; (i < num && i < DC_I2C_BUS_MAX); i++) { + val->i2c_id = (u32)i2c->id; + val->use = true; + val++; + i2c++; + } + + kfree(vbios_i2c); + return ret; +} + +static bool parse_vbios_crtc(struct desc_node *this, struct vbios_cmd *cmd) +{ + bool ret = true; + u64 request = (u64)cmd->req; + u32 *val = (u32 *)cmd->res; + struct vbios_crtc crtc; + + memset(&crtc, 0xff, sizeof(crtc)); + memcpy(&crtc, this->data, min_t(u32, this->desc->size, sizeof(crtc))); + + switch (request) { + case VBIOS_CRTC_ID: + *val = crtc.crtc_id; + break; + case VBIOS_CRTC_ENCODER_ID: + *val = crtc.encoder_id; + break; + case VBIOS_CRTC_MAX_FREQ: + *val = crtc.max_freq; + break; + case VBIOS_CRTC_MAX_WIDTH: + *val = crtc.max_width; + break; + case VBIOS_CRTC_MAX_HEIGHT: + *val = crtc.max_height; + break; + case VBIOS_CRTC_IS_VB_TIMING: + *val = crtc.is_vb_timing; + break; + default: + ret = false; + break; + } + + return ret; +} + +static bool parse_vbios_encoder(struct desc_node *this, struct vbios_cmd *cmd) +{ + bool ret = true; + u64 request = (u64)cmd->req; + u32 *val = (u32 *)cmd->res; + struct vbios_encoder encoder; + + memset(&encoder, 0xff, sizeof(encoder)); + memcpy(&encoder, this->data, + min_t(u32, this->desc->size, sizeof(encoder))); + + switch (request) { + case VBIOS_ENCODER_I2C_ID: + *val = encoder.i2c_id; + break; + case VBIOS_ENCODER_CONNECTOR_ID: + *val = encoder.connector_id; + break; + case VBIOS_ENCODER_TYPE: + *val = encoder.type; + break; + case VBIOS_ENCODER_CONFIG_TYPE: + *val = encoder.config_type; + break; + case VBIOS_ENCODER_CHIP: + *val = encoder.chip; + break; + case VBIOS_ENCODER_CHIP_ADDR: + *val = encoder.chip_addr; + break; + default: + ret = false; + break; + } + + return ret; +} + +static bool parse_vbios_cfg_encoder(struct desc_node *this, + struct vbios_cmd *cmd) +{ + bool ret = true; + u64 request = (u64)cmd->req; + u32 *val = (u32 *)cmd->res; + struct cfg_encoder *cfg_encoder; + struct cfg_encoder *cfg; + struct vbios_cfg_encoder *vbios_cfg_encoder; + u32 num, size, i = 0; + + vbios_cfg_encoder = (struct vbios_cfg_encoder *)this->data; + size = sizeof(struct vbios_cfg_encoder); + num = this->desc->size / size; + + switch (request) { + case VBIOS_ENCODER_CONFIG_PARAM: + cfg_encoder = (struct cfg_encoder *)kzalloc( + sizeof(struct cfg_encoder) * num, GFP_KERNEL); + cfg = cfg_encoder; + for (i = 0; i < num; i++) { + cfg->reg_num = vbios_cfg_encoder->reg_num; + cfg->hdisplay = vbios_cfg_encoder->hdisplay; + cfg->vdisplay = vbios_cfg_encoder->vdisplay; + memcpy(&cfg->config_regs, + &vbios_cfg_encoder->config_regs, + sizeof(struct vbios_conf_reg) * 256); + + cfg++; + vbios_cfg_encoder++; + } + cmd->res = (void *)cfg_encoder; + break; + case VBIOS_ENCODER_CONFIG_NUM: + *val = num; + break; + default: + ret = false; + break; + } + + return ret; +} + +static bool parse_vbios_connector(struct desc_node *this, struct vbios_cmd *cmd) +{ + bool ret = true; + u64 request = (u64)cmd->req; + u32 *val = (u32 *)cmd->res; + struct vbios_connector connector; + + memset(&connector, 0xff, sizeof(connector)); + memcpy(&connector, this->data, + min_t(u32, this->desc->size, sizeof(connector))); + + switch (request) { + case VBIOS_CONNECTOR_I2C_ID: + *val = connector.i2c_id; + break; + case VBIOS_CONNECTOR_INTERNAL_EDID: + memcpy((u8 *)(ulong)val, connector.internal_edid, + EDID_LENGTH * 2); + break; + case VBIOS_CONNECTOR_TYPE: + *val = connector.type; + break; + case VBIOS_CONNECTOR_HOTPLUG: + *val = connector.hotplug; + break; + case VBIOS_CONNECTOR_EDID_METHOD: + *val = connector.edid_method; + break; + case VBIOS_CONNECTOR_IRQ_GPIO: + *val = connector.irq_gpio; + break; + case VBIOS_CONNECTOR_IRQ_PLACEMENT: + *val = connector.gpio_placement; + break; + default: + ret = false; + break; + } + + return ret; +} + +static bool parse_vbios_backlight(struct desc_node *this, struct vbios_cmd *cmd) +{ + return 0; +} + +static bool parse_vbios_pwm(struct desc_node *this, struct vbios_cmd *cmd) +{ + bool ret = true; + u64 request = (u64)cmd->req; + u32 *val = (u32 *)cmd->res; + struct vbios_pwm *pwm = (struct vbios_pwm *)this->data; + + switch (request) { + case VBIOS_PWM_ID: + *val = pwm->pwm; + break; + case VBIOS_PWM_PERIOD: + *val = pwm->peroid; + break; + case VBIOS_PWM_POLARITY: + *val = pwm->polarity; + break; + default: + ret = false; + break; + } + + return ret; +} + +static bool parse_vbios_header(struct desc_node *this, struct vbios_cmd *cmd) +{ + return true; +} + +static bool parse_vbios_default(struct desc_node *this, struct vbios_cmd *cmd) +{ + struct vbios_desc *vb_desc; + + vb_desc = this->desc; + DRM_WARN("Current descriptor[T-%d][V-%d] cannot be interprete.\n", + vb_desc->type, vb_desc->ver); + return false; +} + +#define FUNC(t, v, f) \ + { \ + .type = t, .ver = v, .func = f, \ + } + +static struct desc_func tables[] = { + FUNC(desc_header, ver_v1, parse_vbios_header), + FUNC(desc_i2c, ver_v1, parse_vbios_i2c), + FUNC(desc_crtc, ver_v1, parse_vbios_crtc), + FUNC(desc_encoder, ver_v1, parse_vbios_encoder), + FUNC(desc_connector, ver_v1, parse_vbios_connector), + FUNC(desc_cfg_encoder, ver_v1, parse_vbios_cfg_encoder), + FUNC(desc_backlight, ver_v1, parse_vbios_backlight), + FUNC(desc_pwm, ver_v1, parse_vbios_pwm), +}; + +static inline parse_func *get_parse_func(struct vbios_desc *vb_desc) +{ + int i; + u32 type = vb_desc->type; + u32 ver = vb_desc->ver; + parse_func *func = parse_vbios_default; + u32 tt_num = ARRAY_SIZE(tables); + + for (i = 0; i < tt_num; i++) { + if ((tables[i].ver == ver) && (tables[i].type == type)) { + func = tables[i].func; + break; + } + } + + return func; +} + +static inline u32 insert_desc_list(struct loongson_drm_device *ldev, + struct vbios_desc *vb_desc) +{ + struct desc_node *node; + parse_func *func = NULL; + + WARN_ON(!ldev || !vb_desc); + node = (struct desc_node *)kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + func = get_parse_func(vb_desc); + node->parse = func; + node->desc = (void *)vb_desc; + node->data = ((u8 *)ldev->vbios + vb_desc->offset); + list_add_tail(&node->head, &ldev->desc_list); + + return 0; +} + +static inline void free_desc_list(struct loongson_drm_device *ldev) +{ + struct desc_node *node, *tmp; + + list_for_each_entry_safe (node, tmp, &ldev->desc_list, head) { + list_del(&node->head); + kfree(node); + } +} + +#define DESC_LIST_MAX 1024 + +static u32 parse_vbios_desc(struct loongson_drm_device *ldev) +{ + u32 i, ret = 0; + struct vbios_desc *desc; + enum desc_type type = 0; + u8 *vbios = (u8 *)ldev->vbios; + + WARN_ON(!vbios); + + desc = (struct vbios_desc *)(vbios + VBIOS_DESC_OFFSET); + for (i = 0; i < DESC_LIST_MAX; i++) { + type = desc->type; + if (type == desc_max) + break; + + ret = insert_desc_list(ldev, desc); + if (ret) + DRM_DEBUG_KMS("Parse T-%d V-%d failed[%d]\n", desc->ver, + desc->type, ret); + + desc++; + } + + return ret; +} + +static inline struct desc_node *get_desc_node(struct loongson_drm_device *ldev, + u16 type, u8 index) +{ + struct desc_node *node, *tmp; + struct vbios_desc *vb_desc; + + list_for_each_entry_safe (node, tmp, &ldev->desc_list, head) { + vb_desc = node->desc; + if (vb_desc->type == type && vb_desc->index == index) + return node; + } + + return NULL; +} + +static bool vbios_get_data(struct loongson_drm_device *ldev, struct vbios_cmd *cmd) +{ + struct desc_node *node; + + WARN_ON(!cmd); + + node = get_desc_node(ldev, cmd->type, cmd->index); + if (node && node->parse) + return node->parse(node, cmd); + + DRM_DEBUG_DRIVER("Failed to get node(%d,%d)\n", cmd->type, cmd->index); + + return false; +} + +u32 get_connector_type(struct loongson_drm_device *ldev, u32 index) +{ + u32 type = -1; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_TYPE; + vbt_cmd.res = (void *)(ulong)&type; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + type = -1; + + return type; +} + +u16 get_connector_i2cid(struct loongson_drm_device *ldev, u32 index) +{ + u16 i2c_id = -1; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_I2C_ID; + vbt_cmd.res = (void *)(ulong)&i2c_id; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + i2c_id = -1; + + return i2c_id; +} + +u32 get_connector_irq_gpio(struct loongson_drm_device *ldev, u32 index) +{ + int ret; + u32 irq_gpio; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_IRQ_GPIO; + vbt_cmd.res = (void *)(ulong)&irq_gpio; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + return -1; + + return irq_gpio; +} + +enum gpio_placement get_connector_gpio_placement(struct loongson_drm_device *ldev, + u32 index) +{ + int ret; + enum gpio_placement irq_placement; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_IRQ_PLACEMENT; + vbt_cmd.res = (void *)(ulong)&irq_placement; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + return -1; + + return irq_placement; +} + +u16 get_hotplug_mode(struct loongson_drm_device *ldev, u32 index) +{ + u16 mode = -1; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_HOTPLUG; + vbt_cmd.res = (void *)(ulong)&mode; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + mode = -1; + + return mode; +} + +u16 get_edid_method(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u16 method = via_null; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_EDID_METHOD; + vbt_cmd.res = (void *)(ulong)&method; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + method = via_null; + + return method; +} + +u8 *get_vbios_edid(struct loongson_drm_device *ldev, u32 index) +{ + u8 *edid = NULL; + bool ret = false; + struct vbios_cmd vbt_cmd; + + edid = kzalloc(sizeof(u8) * EDID_LENGTH * 2, GFP_KERNEL); + if (!edid) + return edid; + + vbt_cmd.index = index; + vbt_cmd.type = desc_connector; + vbt_cmd.req = (void *)(ulong)VBIOS_CONNECTOR_INTERNAL_EDID; + vbt_cmd.res = (void *)(ulong)edid; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + return NULL; + + return edid; +} + +u32 get_vbios_pwm(struct loongson_drm_device *ldev, u32 index, u16 request) +{ + u32 value = -1; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_pwm; + vbt_cmd.req = (void *)(ulong)request; + vbt_cmd.res = (void *)(ulong)&value; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + value = 0xffffffff; + + return value; +} + +u32 get_crtc_id(struct loongson_drm_device *ldev, u32 index) +{ + u32 crtc_id = 0; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_ID; + vbt_cmd.res = (void *)(ulong)&crtc_id; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + crtc_id = 0; + + return crtc_id; +} + +u32 get_crtc_max_freq(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u32 max_freq = 0; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_MAX_FREQ; + vbt_cmd.res = (void *)(ulong)&max_freq; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + max_freq = 0; + + return max_freq; +} + +u32 get_crtc_max_width(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u32 max_width = 0; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_MAX_WIDTH; + vbt_cmd.res = (void *)(ulong)&max_width; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + max_width = LOONGSON_MAX_FB_WIDTH; + + return max_width; +} + +u32 get_crtc_max_height(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u32 max_height = 0; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_MAX_HEIGHT; + vbt_cmd.res = (void *)(ulong)&max_height; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + max_height = LOONGSON_MAX_FB_HEIGHT; + + return max_height; +} + +u32 get_crtc_encoder_id(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u32 encoder_id = 0; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_ENCODER_ID; + vbt_cmd.res = (void *)(ulong)&encoder_id; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + encoder_id = 0; + + return encoder_id; +} + +bool get_crtc_is_vb_timing(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + bool vb_timing = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_crtc; + vbt_cmd.req = (void *)(ulong)VBIOS_CRTC_IS_VB_TIMING; + vbt_cmd.res = (void *)(ulong)&vb_timing; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + vb_timing = false; + + return vb_timing; +} + +struct crtc_timing *get_crtc_timing(struct loongson_drm_device *ldev, u32 index) +{ + return NULL; +} + +u32 get_encoder_connector_id(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + u32 connector_id = 0; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CONNECTOR_ID; + vbt_cmd.res = (void *)(ulong)&connector_id; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + connector_id = 0; + + return connector_id; +} + +u32 get_encoder_i2c_id(struct loongson_drm_device *ldev, u32 index) +{ + u32 i2c_id = 0; + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_I2C_ID; + vbt_cmd.res = (void *)(ulong)&i2c_id; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + i2c_id = 0; + + return i2c_id; +} + +struct cfg_encoder *get_encoder_config(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + struct vbios_cmd vbt_cmd; + struct cfg_encoder *encoder_config = NULL; + + vbt_cmd.index = index; + vbt_cmd.type = desc_cfg_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CONFIG_PARAM; + ret = vbios_get_data(ldev, &vbt_cmd); + if (ret) + encoder_config = (struct cfg_encoder *)vbt_cmd.res; + + return encoder_config; +} + +u32 get_encoder_cfg_num(struct loongson_drm_device *ldev, u32 index) +{ + struct vbios_cmd vbt_cmd; + bool ret = false; + u32 cfg_num = 0; + + vbt_cmd.index = index; + vbt_cmd.type = desc_cfg_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CONFIG_NUM; + vbt_cmd.res = (void *)(ulong)&cfg_num; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + cfg_num = 0; + + return cfg_num; +} + +enum encoder_config get_encoder_config_type(struct loongson_drm_device *ldev, + u32 index) +{ + bool ret = false; + enum encoder_config config_type = encoder_bios_config; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CONFIG_TYPE; + vbt_cmd.res = (void *)(ulong)&config_type; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + config_type = encoder_bios_config; + + return config_type; +} + +enum encoder_object get_encoder_chip(struct loongson_drm_device *ldev, u32 index) +{ + int ret; + enum encoder_object chip; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CHIP; + vbt_cmd.res = (void *)(ulong)&chip; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + return Unknown; + + return chip; +} + +u8 get_encoder_chip_addr(struct loongson_drm_device *ldev, u32 index) +{ + int ret; + u8 chip_addr; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_CHIP_ADDR; + vbt_cmd.res = (void *)(ulong)&chip_addr; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + return Unknown; + + return chip_addr; +} + +enum encoder_type get_encoder_type(struct loongson_drm_device *ldev, u32 index) +{ + bool ret = false; + enum encoder_type type = encoder_dac; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = index; + vbt_cmd.type = desc_encoder; + vbt_cmd.req = (void *)(ulong)VBIOS_ENCODER_TYPE; + vbt_cmd.res = (void *)(ulong)&type; + ret = vbios_get_data(ldev, &vbt_cmd); + if (!ret) + type = encoder_dac; + + return type; +} + +bool get_loongson_i2c(struct loongson_drm_device *ldev) +{ + bool ret = false; + struct vbios_cmd vbt_cmd; + + vbt_cmd.index = 0; + vbt_cmd.type = desc_i2c; + vbt_cmd.res = (void *)&ldev->i2c_bus; + ret = vbios_get_data(ldev, &vbt_cmd); + + if (!ret) { + ldev->i2c_bus[0].use = true; + ldev->i2c_bus[1].use = true; + } + + return true; +} + +static void *get_vbios_from_acpi(struct loongson_drm_device *ldev) +{ + void *vbios = NULL; +#ifdef CONFIG_ACPI + struct acpi_viat_table *viat; + struct acpi_table_header *hdr; + + if (!ACPI_SUCCESS(acpi_get_table("VIAT", 1, &hdr))) + return NULL; + + if (hdr->length != sizeof(struct acpi_viat_table)) { + DRM_WARN("ACPI VIAT table present but broken (length error)\n"); + return NULL; + } + + vbios = kmalloc(VBIOS_SIZE, GFP_KERNEL); + if (!vbios) + return NULL; + + viat = (struct acpi_viat_table *)hdr; + memcpy(vbios, phys_to_virt(viat->vbios_addr), VBIOS_SIZE); + + DRM_INFO("Get VBIOS from ACPI success!\n"); +#endif + return vbios; +} + +static void *get_vbios_from_vram(struct loongson_drm_device *ldev) +{ + void *bios, *vbios = NULL; + u64 vram_base = pci_resource_start(ldev->gpu_pdev, 2); + u64 vram_size = pci_resource_len(ldev->gpu_pdev, 2); + u64 vbios_addr = vram_base + vram_size - VBIOS_OFFSET; + + bios = ioremap(vbios_addr, VBIOS_SIZE); + if (!bios) + return NULL; + + vbios = kmalloc(VBIOS_SIZE, GFP_KERNEL); + if (!vbios) + return NULL; + + memcpy(vbios, bios, VBIOS_SIZE); + + iounmap(bios); + + if (memcmp(vbios, VBIOS_TITLE, strlen(VBIOS_TITLE))) { + kfree(vbios); + return NULL; + } + + DRM_INFO("Get VBIOS from VRAM success!\n"); + + return vbios; +} + +bool loongson_vbios_init(struct loongson_drm_device *ldev) +{ + void *vbios = NULL; + struct loongson_vbios *header; + + vbios = get_vbios_from_vram(ldev); + if (vbios) + goto success; + + vbios = get_vbios_from_acpi(ldev); + if (vbios) + goto success; + + vbios = kzalloc(256 * 1024, GFP_KERNEL); + if (!vbios) + return false; + + header = vbios; + header->crtc_num = 2; + +success: + header = ldev->vbios = vbios; + ldev->num_crtc = header->crtc_num; + + DRM_INFO("Loongson VBIOS version %d.%d\n", header->version_major, + header->version_minor); + + INIT_LIST_HEAD(&ldev->desc_list); + parse_vbios_desc(ldev); + + return true; +} + +void loongson_vbios_exit(struct loongson_drm_device *ldev) +{ + free_desc_list(ldev); + kfree(ldev->vbios); +} diff --git a/drivers/gpu/drm/loongson/loongson_vbios.h b/drivers/gpu/drm/loongson/loongson_vbios.h new file mode 100644 index 0000000000000000000000000000000000000000..25ae943a6c958a67566c45869db7a350abc0f539 --- /dev/null +++ b/drivers/gpu/drm/loongson/loongson_vbios.h @@ -0,0 +1,283 @@ +#ifndef __LOONGSON_VBIOS_H__ +#define __LOONGSON_VBIOS_H__ + +#define VBIOS_PWM_ID 0x0 +#define VBIOS_PWM_PERIOD 0x1 +#define VBIOS_PWM_POLARITY 0x2 + +#define VBIOS_CRTC_ID 0x1 +#define VBIOS_CRTC_ENCODER_ID 0x2 +#define VBIOS_CRTC_MAX_FREQ 0x3 +#define VBIOS_CRTC_MAX_WIDTH 0x4 +#define VBIOS_CRTC_MAX_HEIGHT 0x5 +#define VBIOS_CRTC_IS_VB_TIMING 0x6 + +#define VBIOS_ENCODER_I2C_ID 0x1 +#define VBIOS_ENCODER_CONNECTOR_ID 0x2 +#define VBIOS_ENCODER_TYPE 0x3 +#define VBIOS_ENCODER_CONFIG_TYPE 0x4 +#define VBIOS_ENCODER_CONFIG_PARAM 0x1 +#define VBIOS_ENCODER_CONFIG_NUM 0x2 +#define VBIOS_ENCODER_CHIP 0x05 +#define VBIOS_ENCODER_CHIP_ADDR 0x06 + +#define VBIOS_CONNECTOR_I2C_ID 0x1 +#define VBIOS_CONNECTOR_INTERNAL_EDID 0x2 +#define VBIOS_CONNECTOR_TYPE 0x3 +#define VBIOS_CONNECTOR_HOTPLUG 0x4 +#define VBIOS_CONNECTOR_EDID_METHOD 0x5 +#define VBIOS_CONNECTOR_IRQ_PLACEMENT 0x06 +#define VBIOS_CONNECTOR_IRQ_GPIO 0x07 + +struct desc_node; +struct vbios_cmd; +typedef bool(parse_func)(struct desc_node *, struct vbios_cmd *); + +enum desc_type { + desc_header = 0, + desc_crtc, + desc_encoder, + desc_connector, + desc_i2c, + desc_pwm, + desc_gpio, + desc_backlight, + desc_fan, + desc_irq_vblank, + desc_cfg_encoder, + desc_max = 0xffff +}; + +enum desc_ver { + ver_v1, +}; + +enum hotplug { + disable = 0, + polling, + irq, + hotplug_max = 0xffffffff, +}; + +enum loongson_edid_method { + via_null = 0, + via_i2c, + via_vbios, + via_encoder, + via_max = 0xffffffff, +}; + +enum i2c_type { i2c_cpu, i2c_gpio, i2c_max = -1 }; + +enum vbios_backlight_type { bl_unuse, bl_ec, bl_pwm }; + +enum encoder_config { + encoder_transparent = 0, + encoder_os_config, + encoder_bios_config, + encoder_timing_filling, + encoder_kernel_driver, + encoder_type_max = 0xffffffff, +}; + +enum encoder_type { + encoder_none, + encoder_dac, + encoder_tmds, + encoder_lvds, + encoder_tvdac, + encoder_virtual, + encoder_dsi, + encoder_dpmst, + encoder_dpi +}; + +enum connector_type { + connector_unknown, + connector_vga, + connector_dvii, + connector_dvid, + connector_dvia, + connector_composite, + connector_svideo, + connector_lvds, + connector_component, + connector_9pindin, + connector_displayport, + connector_hdmia, + connector_hdmib, + connector_tv, + connector_edp, + connector_virtual, + connector_dsi, + connector_dpi +}; + +enum gpio_placement { + GPIO_PLACEMENT_LS3A = 0, + GPIO_PLACEMENT_LS7A, +}; + +enum encoder_object { + Unknown = 0x00, + INTERNAL_DVO = 0x01, + INTERNAL_HDMI = 0x02, + VGA_CH7055 = 0x10, + VGA_ADV7125 = 0x11, + DVI_TFP410 = 0x20, + HDMI_IT66121 = 0x30, + HDMI_SIL9022 = 0x31, + HDMI_LT8618 = 0x32, + HDMI_MS7210 = 0x33, + EDP_NCS8805 = 0x40 +}; + +struct loongson_vbios { + char title[16]; + uint32_t version_major; + uint32_t version_minor; + char information[20]; + uint32_t crtc_num; + uint32_t crtc_offset; + uint32_t connector_num; + uint32_t connector_offset; + uint32_t encoder_num; + uint32_t encoder_offset; +} __packed; + +struct vbios_header { + u32 feature; + u8 oem_vendor[32]; + u8 oem_product[32]; + u32 legacy_offset; + u32 legacy_size; + u32 desc_offset; + u32 desc_size; + u32 data_offset; + u32 data_size; +} __packed; + +struct vbios_backlight { + u32 feature; + u8 used; + enum vbios_backlight_type type; +} __packed; + +struct vbios_i2c { + u32 feature; + u16 id; + enum i2c_type type; +} __packed; + +struct vbios_pwm { + u32 feature; + u8 pwm; + u8 polarity; + u32 peroid; +} __packed; + +struct vbios_desc { + u16 type; + u8 ver; + u8 index; + u32 offset; + u32 size; + u64 ext[2]; +} __packed; + +struct vbios_cmd { + u8 index; + enum desc_type type; + u64 *req; + void *res; +}; + +struct desc_func { + enum desc_type type; + u16 ver; + s8 *name; + u8 index; + parse_func *func; +}; + +struct desc_node { + struct list_head head; + u8 *data; + struct vbios_desc *desc; + parse_func *parse; +}; + +struct vbios_crtc { + u32 feature; + u32 crtc_id; + u32 encoder_id; + u32 max_freq; + u32 max_width; + u32 max_height; + bool is_vb_timing; +} __packed; + +struct vbios_encoder { + u32 feature; + u32 i2c_id; + u32 connector_id; + enum encoder_type type; + enum encoder_config config_type; + enum encoder_object chip; + u8 chip_addr; +} __packed; + +struct vbios_connector { + u32 feature; + u32 i2c_id; + u8 internal_edid[256]; + enum connector_type type; + enum hotplug hotplug; + enum loongson_edid_method edid_method; + u32 irq_gpio; + enum gpio_placement gpio_placement; +} __packed; + +struct vbios_conf_reg { + u8 dev_addr; + u8 reg; + u8 value; +} __packed; + +struct vbios_cfg_encoder { + u32 hdisplay; + u32 vdisplay; + u8 reg_num; + struct vbios_conf_reg config_regs[256]; +} __packed; + +bool loongson_vbios_init(struct loongson_drm_device *ldev); +void loongson_vbios_exit(struct loongson_drm_device *ldev); +u32 get_connector_type(struct loongson_drm_device *ldev, u32 index); +u16 get_connector_i2cid(struct loongson_drm_device *ldev, u32 index); +u16 get_hotplug_mode(struct loongson_drm_device *ldev, u32 index); +u8 *get_vbios_edid(struct loongson_drm_device *ldev, u32 index); +u16 get_edid_method(struct loongson_drm_device *ldev, u32 index); +u32 get_vbios_pwm(struct loongson_drm_device *ldev, u32 index, u16 request); +u32 get_crtc_id(struct loongson_drm_device *ldev, u32 index); +u32 get_crtc_max_freq(struct loongson_drm_device *ldev, u32 index); +u32 get_crtc_max_width(struct loongson_drm_device *ldev, u32 index); +u32 get_crtc_max_height(struct loongson_drm_device *ldev, u32 index); +u32 get_crtc_encoder_id(struct loongson_drm_device *ldev, u32 index); +bool get_crtc_is_vb_timing(struct loongson_drm_device *ldev, u32 index); + +struct crtc_timing *get_crtc_timing(struct loongson_drm_device *ldev, u32 index); + +bool get_loongson_i2c(struct loongson_drm_device *ldev); +u32 get_encoder_i2c_id(struct loongson_drm_device *ldev, u32 index); +u32 get_encoder_connector_id(struct loongson_drm_device *ldev, u32 index); +enum encoder_config get_encoder_config_type(struct loongson_drm_device *ldev, u32 index); +enum encoder_type get_encoder_type(struct loongson_drm_device *ldev, u32 index); +struct cfg_encoder *get_encoder_config(struct loongson_drm_device *ldev, u32 index); +u32 get_encoder_cfg_num(struct loongson_drm_device *ldev, u32 index); +enum encoder_object get_encoder_chip(struct loongson_drm_device *ldev, u32 index); +u8 get_encoder_chip_addr(struct loongson_drm_device *ldev, u32 index); +u32 get_connector_irq_gpio(struct loongson_drm_device *ldev, u32 index); +enum gpio_placement get_connector_gpio_placement(struct loongson_drm_device *ldev, + u32 index); +#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_vmm.c b/drivers/gpu/drm/nouveau/nouveau_vmm.c index a49e88129c922ae4510a62e6b05815f6d878516f..ce1d53b8597f5bba7e64e97bf785d042ff9057c8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_vmm.c @@ -108,6 +108,9 @@ nouveau_vma_new(struct nouveau_bo *nvbo, struct nouveau_vmm *vmm, } else { ret = nvif_vmm_get(&vmm->vmm, PTES, false, mem->mem.page, 0, mem->mem.size, &tmp); + if (ret) + goto done; + vma->addr = tmp.addr; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index 02c4eb28cef44db11ef989f34cab124b74d4f84b..f3db07325dbd899c70d54ed519b09b2339e86bbc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -221,8 +221,11 @@ nv50_instobj_acquire(struct nvkm_memory *memory) void __iomem *map = NULL; /* Already mapped? */ - if (refcount_inc_not_zero(&iobj->maps)) + if (refcount_inc_not_zero(&iobj->maps)) { + /* read barrier match the wmb on refcount set */ + smp_rmb(); return iobj->map; + } /* Take the lock, and re-check that another thread hasn't * already mapped the object in the meantime. @@ -249,6 +252,8 @@ nv50_instobj_acquire(struct nvkm_memory *memory) iobj->base.memory.ptrs = &nv50_instobj_fast; else iobj->base.memory.ptrs = &nv50_instobj_slow; + /* barrier to ensure the ptrs are written before refcount is set */ + smp_wmb(); refcount_set(&iobj->maps, 1); } diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index ee01b61a6bafa6f322097272040b55e93fb46c64..51470020ba61d6a043de06aa24dba2f29a558ab9 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -3635,6 +3635,7 @@ static const struct panel_desc tianma_tm070jdhg30 = { }, .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG, .connector_type = DRM_MODE_CONNECTOR_LVDS, + .bus_flags = DRM_BUS_FLAG_DE_HIGH, }; static const struct panel_desc tianma_tm070jvhg33 = { @@ -3647,6 +3648,7 @@ static const struct panel_desc tianma_tm070jvhg33 = { }, .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG, .connector_type = DRM_MODE_CONNECTOR_LVDS, + .bus_flags = DRM_BUS_FLAG_DE_HIGH, }; static const struct display_timing tianma_tm070rvhg71_timing = { diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index 13f25ec1fe25cdb4e1c6e6f5a35be15b38bf97e3..4d1c4a2bf909ba07f095701f26eca55e9934b239 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -8108,6 +8108,7 @@ int cik_irq_process(struct radeon_device *rdev) if (queue_thermal) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; + WREG32(IH_RB_RPTR, rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 061ef6c0085921e9661c983be1c1b53fcdfd59d5..96dbf595254ca7f73884294069a465f6815dbf66 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -4920,6 +4920,7 @@ int evergreen_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; + WREG32(IH_RB_RPTR, rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index d9a33ca768f345c2dcb34aa0bec01e705945d92a..cd5418ccf20ee2a00076eb760280e0ee7a3799b7 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -4331,6 +4331,7 @@ int r600_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; + WREG32(IH_RB_RPTR, rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c index 495700d16fc9298e299092dc1e65eb2dcc6a81c5..c551420268c75e6f87b9ac24280f67d4ba7fcf76 100644 --- a/drivers/gpu/drm/radeon/radeon_asic.c +++ b/drivers/gpu/drm/radeon/radeon_asic.c @@ -2477,6 +2477,9 @@ int radeon_asic_init(struct radeon_device *rdev) if (rdev->family == CHIP_HAINAN) { rdev->has_uvd = false; rdev->has_vce = false; + } else if (rdev->family == CHIP_OLAND) { + rdev->has_uvd = true; + rdev->has_vce = false; } else { rdev->has_uvd = true; rdev->has_vce = true; diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 37093cea24c59d2fcf028572b44975a446cd5b3b..a5706f144b3abe3aae732a4302be5be9ac801993 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -184,6 +184,7 @@ void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring, if (hdp_flush && rdev->asic->mmio_hdp_flush) rdev->asic->mmio_hdp_flush(rdev); radeon_ring_set_wptr(rdev, ring); + mmiowb(); /* Make sure wptr is up-to-date for hw */ } /** diff --git a/drivers/gpu/drm/radeon/radeon_vce.c b/drivers/gpu/drm/radeon/radeon_vce.c index 5e80064447046961e12f7da226a53ddacf82ef59..7f12bef4f562c6f995bd788d9d414c5f8e5d2767 100644 --- a/drivers/gpu/drm/radeon/radeon_vce.c +++ b/drivers/gpu/drm/radeon/radeon_vce.c @@ -68,7 +68,6 @@ int radeon_vce_init(struct radeon_device *rdev) case CHIP_TAHITI: case CHIP_PITCAIRN: case CHIP_VERDE: - case CHIP_OLAND: case CHIP_ARUBA: fw_name = FIRMWARE_TAHITI; break; diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 27b14eff532cb8664162ed518bb2a602ab60c181..cb75ff1f6f2cec21a4275a872f81cc9405d529d3 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -1206,13 +1206,17 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) r = radeon_bo_create(rdev, pd_size, align, true, RADEON_GEM_DOMAIN_VRAM, 0, NULL, NULL, &vm->page_directory); - if (r) + if (r) { + kfree(vm->page_tables); + vm->page_tables = NULL; return r; - + } r = radeon_vm_clear_bo(rdev, vm->page_directory); if (r) { radeon_bo_unref(&vm->page_directory); vm->page_directory = NULL; + kfree(vm->page_tables); + vm->page_tables = NULL; return r; } diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index e2eac766666e2476a546342f75950c21bec91d36..c6f8e8edb8eab7da7d2dee8eb1f6423d9a35d225 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -6447,6 +6447,7 @@ int si_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; + WREG32(IH_RB_RPTR, rptr); atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/vce_v1_0.c b/drivers/gpu/drm/radeon/vce_v1_0.c index bd75bbcf5bf6368dcb3623aba159fd67f0eb2004..028213eabd9440a7b5353e66a82702f625e47629 100644 --- a/drivers/gpu/drm/radeon/vce_v1_0.c +++ b/drivers/gpu/drm/radeon/vce_v1_0.c @@ -168,7 +168,6 @@ int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data) chip_id = 0x01000015; break; case CHIP_PITCAIRN: - case CHIP_OLAND: chip_id = 0x01000016; break; case CHIP_ARUBA: diff --git a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c index 3c5744a91d4a006fc43f86f11a2729d6055c1bc1..26fd2761e80db3d02585a0be0c8cc2cfa4c7d165 100644 --- a/drivers/gpu/drm/tidss/tidss_crtc.c +++ b/drivers/gpu/drm/tidss/tidss_crtc.c @@ -168,13 +168,13 @@ static void tidss_crtc_atomic_flush(struct drm_crtc *crtc, struct tidss_device *tidss = to_tidss(ddev); unsigned long flags; - dev_dbg(ddev->dev, - "%s: %s enabled %d, needs modeset %d, event %p\n", __func__, - crtc->name, drm_atomic_crtc_needs_modeset(crtc->state), - crtc->state->enable, crtc->state->event); + dev_dbg(ddev->dev, "%s: %s is %sactive, %s modeset, event %p\n", + __func__, crtc->name, crtc->state->active ? "" : "not ", + drm_atomic_crtc_needs_modeset(crtc->state) ? "needs" : "doesn't need", + crtc->state->event); /* There is nothing to do if CRTC is not going to be enabled. */ - if (!crtc->state->enable) + if (!crtc->state->active) return; /* diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 1968df9743fcbf59a6aa58e8e6aac204ea0c4db2..293d9d926f96e540f7e63b1af30d3d0d6beef0a6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -397,7 +397,7 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) tmp = pgprot_noncached(tmp); #endif #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ - defined(__powerpc__) || defined(__mips__) + defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) if (caching_flags & TTM_PL_FLAG_WC) tmp = pgprot_writecombine(tmp); else diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 476967ab6294c8033737c87b38369b0e1ba954f9..5281d693b32d2d365ebd09a7206064ad18005499 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -1446,7 +1446,6 @@ static void implement(const struct hid_device *hid, u8 *report, hid_warn(hid, "%s() called with too large value %d (n: %d)! (%s)\n", __func__, value, n, current->comm); - WARN_ON(1); value &= m; } } diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index 1b5e5e9f577dbcd71e9f2ad2d1064ea9916baaf1..726a5d76615d2015c17fa6df7ac43b9f1f6b87eb 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -2612,8 +2612,8 @@ static void wacom_wac_finger_slot(struct wacom_wac *wacom_wac, { struct hid_data *hid_data = &wacom_wac->hid_data; bool mt = wacom_wac->features.touch_max > 1; - bool prox = hid_data->tipswitch && - report_touch_events(wacom_wac); + bool touch_down = hid_data->tipswitch && hid_data->confidence; + bool prox = touch_down && report_touch_events(wacom_wac); if (wacom_wac->shared->has_mute_touch_switch && !wacom_wac->shared->is_touch_on) { @@ -2652,24 +2652,6 @@ static void wacom_wac_finger_slot(struct wacom_wac *wacom_wac, } } -static bool wacom_wac_slot_is_active(struct input_dev *dev, int key) -{ - struct input_mt *mt = dev->mt; - struct input_mt_slot *s; - - if (!mt) - return false; - - for (s = mt->slots; s != mt->slots + mt->num_slots; s++) { - if (s->key == key && - input_mt_get_value(s, ABS_MT_TRACKING_ID) >= 0) { - return true; - } - } - - return false; -} - static void wacom_wac_finger_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -2717,14 +2699,8 @@ static void wacom_wac_finger_event(struct hid_device *hdev, } if (usage->usage_index + 1 == field->report_count) { - if (equivalent_usage == wacom_wac->hid_data.last_slot_field) { - bool touch_removed = wacom_wac_slot_is_active(wacom_wac->touch_input, - wacom_wac->hid_data.id) && !wacom_wac->hid_data.tipswitch; - - if (wacom_wac->hid_data.confidence || touch_removed) { - wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input); - } - } + if (equivalent_usage == wacom_wac->hid_data.last_slot_field) + wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input); } } diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 05831848b7bf684341471c2b76539d2d03217c9f..fd0969cd7dc6734ccc136a3c0622e34dd7ade513 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -223,8 +223,17 @@ static bool is_ack(struct s3c24xx_i2c *i2c) int tries; for (tries = 50; tries; --tries) { - if (readl(i2c->regs + S3C2410_IICCON) - & S3C2410_IICCON_IRQPEND) { + unsigned long tmp = readl(i2c->regs + S3C2410_IICCON); + + if (!(tmp & S3C2410_IICCON_ACKEN)) { + /* + * Wait a bit for the bus to stabilize, + * delay estimated experimentally. + */ + usleep_range(100, 200); + return true; + } + if (tmp & S3C2410_IICCON_IRQPEND) { if (!(readl(i2c->regs + S3C2410_IICSTAT) & S3C2410_IICSTAT_LASTBIT)) return true; @@ -277,16 +286,6 @@ static void s3c24xx_i2c_message_start(struct s3c24xx_i2c *i2c, stat |= S3C2410_IICSTAT_START; writel(stat, i2c->regs + S3C2410_IICSTAT); - - if (i2c->quirks & QUIRK_POLL) { - while ((i2c->msg_num != 0) && is_ack(i2c)) { - i2c_s3c_irq_nextbyte(i2c, stat); - stat = readl(i2c->regs + S3C2410_IICSTAT); - - if (stat & S3C2410_IICSTAT_ARBITR) - dev_err(i2c->dev, "deal with arbitration loss\n"); - } - } } static inline void s3c24xx_i2c_stop(struct s3c24xx_i2c *i2c, int ret) @@ -693,7 +692,7 @@ static void s3c24xx_i2c_wait_idle(struct s3c24xx_i2c *i2c) static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, struct i2c_msg *msgs, int num) { - unsigned long timeout; + unsigned long timeout = 0; int ret; ret = s3c24xx_i2c_set_master(i2c); @@ -713,16 +712,19 @@ static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, s3c24xx_i2c_message_start(i2c, msgs); if (i2c->quirks & QUIRK_POLL) { - ret = i2c->msg_idx; + while ((i2c->msg_num != 0) && is_ack(i2c)) { + unsigned long stat = readl(i2c->regs + S3C2410_IICSTAT); - if (ret != num) - dev_dbg(i2c->dev, "incomplete xfer (%d)\n", ret); + i2c_s3c_irq_nextbyte(i2c, stat); - goto out; + stat = readl(i2c->regs + S3C2410_IICSTAT); + if (stat & S3C2410_IICSTAT_ARBITR) + dev_err(i2c->dev, "deal with arbitration loss\n"); + } + } else { + timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); } - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); - ret = i2c->msg_idx; /* diff --git a/drivers/iio/adc/ad7091r-base.c b/drivers/iio/adc/ad7091r-base.c index 63b4d6ea4566a022f780a31e7480a11670887bad..76002b91c86a4ae9ecb04c5994e5963cac79c827 100644 --- a/drivers/iio/adc/ad7091r-base.c +++ b/drivers/iio/adc/ad7091r-base.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -28,6 +29,7 @@ #define AD7091R_REG_RESULT_CONV_RESULT(x) ((x) & 0xfff) /* AD7091R_REG_CONF */ +#define AD7091R_REG_CONF_ALERT_EN BIT(4) #define AD7091R_REG_CONF_AUTO BIT(8) #define AD7091R_REG_CONF_CMD BIT(10) @@ -49,6 +51,27 @@ struct ad7091r_state { struct mutex lock; /*lock to prevent concurent reads */ }; +const struct iio_event_spec ad7091r_events[] = { + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_RISING, + .mask_separate = BIT(IIO_EV_INFO_VALUE) | + BIT(IIO_EV_INFO_ENABLE), + }, + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_FALLING, + .mask_separate = BIT(IIO_EV_INFO_VALUE) | + BIT(IIO_EV_INFO_ENABLE), + }, + { + .type = IIO_EV_TYPE_THRESH, + .dir = IIO_EV_DIR_EITHER, + .mask_separate = BIT(IIO_EV_INFO_HYSTERESIS), + }, +}; +EXPORT_SYMBOL_NS_GPL(ad7091r_events, IIO_AD7091R); + static int ad7091r_set_mode(struct ad7091r_state *st, enum ad7091r_mode mode) { int ret, conf; @@ -168,14 +191,148 @@ static int ad7091r_read_raw(struct iio_dev *iio_dev, return ret; } +static int ad7091r_read_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir) +{ + struct ad7091r_state *st = iio_priv(indio_dev); + int val, ret; + + switch (dir) { + case IIO_EV_DIR_RISING: + ret = regmap_read(st->map, + AD7091R_REG_CH_HIGH_LIMIT(chan->channel), + &val); + if (ret) + return ret; + return val != AD7091R_HIGH_LIMIT; + case IIO_EV_DIR_FALLING: + ret = regmap_read(st->map, + AD7091R_REG_CH_LOW_LIMIT(chan->channel), + &val); + if (ret) + return ret; + return val != AD7091R_LOW_LIMIT; + default: + return -EINVAL; + } +} + +static int ad7091r_write_event_config(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, int state) +{ + struct ad7091r_state *st = iio_priv(indio_dev); + + if (state) { + return regmap_set_bits(st->map, AD7091R_REG_CONF, + AD7091R_REG_CONF_ALERT_EN); + } else { + /* + * Set thresholds either to 0 or to 2^12 - 1 as appropriate to + * prevent alerts and thus disable event generation. + */ + switch (dir) { + case IIO_EV_DIR_RISING: + return regmap_write(st->map, + AD7091R_REG_CH_HIGH_LIMIT(chan->channel), + AD7091R_HIGH_LIMIT); + case IIO_EV_DIR_FALLING: + return regmap_write(st->map, + AD7091R_REG_CH_LOW_LIMIT(chan->channel), + AD7091R_LOW_LIMIT); + default: + return -EINVAL; + } + } +} + +static int ad7091r_read_event_value(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + enum iio_event_info info, int *val, int *val2) +{ + struct ad7091r_state *st = iio_priv(indio_dev); + int ret; + + switch (info) { + case IIO_EV_INFO_VALUE: + switch (dir) { + case IIO_EV_DIR_RISING: + ret = regmap_read(st->map, + AD7091R_REG_CH_HIGH_LIMIT(chan->channel), + val); + if (ret) + return ret; + return IIO_VAL_INT; + case IIO_EV_DIR_FALLING: + ret = regmap_read(st->map, + AD7091R_REG_CH_LOW_LIMIT(chan->channel), + val); + if (ret) + return ret; + return IIO_VAL_INT; + default: + return -EINVAL; + } + case IIO_EV_INFO_HYSTERESIS: + ret = regmap_read(st->map, + AD7091R_REG_CH_HYSTERESIS(chan->channel), + val); + if (ret) + return ret; + return IIO_VAL_INT; + default: + return -EINVAL; + } +} + +static int ad7091r_write_event_value(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + enum iio_event_type type, + enum iio_event_direction dir, + enum iio_event_info info, int val, int val2) +{ + struct ad7091r_state *st = iio_priv(indio_dev); + + switch (info) { + case IIO_EV_INFO_VALUE: + switch (dir) { + case IIO_EV_DIR_RISING: + return regmap_write(st->map, + AD7091R_REG_CH_HIGH_LIMIT(chan->channel), + val); + case IIO_EV_DIR_FALLING: + return regmap_write(st->map, + AD7091R_REG_CH_LOW_LIMIT(chan->channel), + val); + default: + return -EINVAL; + } + case IIO_EV_INFO_HYSTERESIS: + return regmap_write(st->map, + AD7091R_REG_CH_HYSTERESIS(chan->channel), + val); + default: + return -EINVAL; + } +} + static const struct iio_info ad7091r_info = { .read_raw = ad7091r_read_raw, + .read_event_config = &ad7091r_read_event_config, + .write_event_config = &ad7091r_write_event_config, + .read_event_value = &ad7091r_read_event_value, + .write_event_value = &ad7091r_write_event_value, }; static irqreturn_t ad7091r_event_handler(int irq, void *private) { - struct ad7091r_state *st = (struct ad7091r_state *) private; - struct iio_dev *iio_dev = dev_get_drvdata(st->dev); + struct iio_dev *iio_dev = private; + struct ad7091r_state *st = iio_priv(iio_dev); unsigned int i, read_val; int ret; s64 timestamp = iio_get_time_ns(iio_dev); @@ -232,9 +389,14 @@ int ad7091r_probe(struct device *dev, const char *name, iio_dev->channels = chip_info->channels; if (irq) { + ret = regmap_update_bits(st->map, AD7091R_REG_CONF, + AD7091R_REG_CONF_ALERT_EN, BIT(4)); + if (ret) + return ret; + ret = devm_request_threaded_irq(dev, irq, NULL, ad7091r_event_handler, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT, name, st); + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, name, iio_dev); if (ret) return ret; } @@ -243,7 +405,14 @@ int ad7091r_probe(struct device *dev, const char *name, if (IS_ERR(st->vref)) { if (PTR_ERR(st->vref) == -EPROBE_DEFER) return -EPROBE_DEFER; + st->vref = NULL; + /* Enable internal vref */ + ret = regmap_set_bits(st->map, AD7091R_REG_CONF, + AD7091R_REG_CONF_INT_VREF); + if (ret) + return dev_err_probe(st->dev, ret, + "Error on enable internal reference\n"); } else { ret = regulator_enable(st->vref); if (ret) @@ -260,7 +429,7 @@ int ad7091r_probe(struct device *dev, const char *name, return devm_iio_device_register(dev, iio_dev); } -EXPORT_SYMBOL_GPL(ad7091r_probe); +EXPORT_SYMBOL_NS_GPL(ad7091r_probe, IIO_AD7091R); static bool ad7091r_writeable_reg(struct device *dev, unsigned int reg) { @@ -290,7 +459,7 @@ const struct regmap_config ad7091r_regmap_config = { .writeable_reg = ad7091r_writeable_reg, .volatile_reg = ad7091r_volatile_reg, }; -EXPORT_SYMBOL_GPL(ad7091r_regmap_config); +EXPORT_SYMBOL_NS_GPL(ad7091r_regmap_config, IIO_AD7091R); MODULE_AUTHOR("Beniamin Bia "); MODULE_DESCRIPTION("Analog Devices AD7091Rx multi-channel converters"); diff --git a/drivers/iio/adc/ad7091r-base.h b/drivers/iio/adc/ad7091r-base.h index 509748aef9b196f1649e419d169b9a04a5081046..b9e1c8bf3440a494ff5e06199d84908838c6ce14 100644 --- a/drivers/iio/adc/ad7091r-base.h +++ b/drivers/iio/adc/ad7091r-base.h @@ -8,6 +8,12 @@ #ifndef __DRIVERS_IIO_ADC_AD7091R_BASE_H__ #define __DRIVERS_IIO_ADC_AD7091R_BASE_H__ +#define AD7091R_REG_CONF_INT_VREF BIT(0) + +/* AD7091R_REG_CH_LIMIT */ +#define AD7091R_HIGH_LIMIT 0xFFF +#define AD7091R_LOW_LIMIT 0x0 + struct device; struct ad7091r_state; @@ -17,6 +23,8 @@ struct ad7091r_chip_info { unsigned int vref_mV; }; +extern const struct iio_event_spec ad7091r_events[3]; + extern const struct regmap_config ad7091r_regmap_config; int ad7091r_probe(struct device *dev, const char *name, diff --git a/drivers/iio/adc/ad7091r5.c b/drivers/iio/adc/ad7091r5.c index 9665679c3ea6dfbf9cb1c038f525d212b908b698..12d475463945d8543d027c91293bca1b1c0c8a40 100644 --- a/drivers/iio/adc/ad7091r5.c +++ b/drivers/iio/adc/ad7091r5.c @@ -12,26 +12,6 @@ #include "ad7091r-base.h" -static const struct iio_event_spec ad7091r5_events[] = { - { - .type = IIO_EV_TYPE_THRESH, - .dir = IIO_EV_DIR_RISING, - .mask_separate = BIT(IIO_EV_INFO_VALUE) | - BIT(IIO_EV_INFO_ENABLE), - }, - { - .type = IIO_EV_TYPE_THRESH, - .dir = IIO_EV_DIR_FALLING, - .mask_separate = BIT(IIO_EV_INFO_VALUE) | - BIT(IIO_EV_INFO_ENABLE), - }, - { - .type = IIO_EV_TYPE_THRESH, - .dir = IIO_EV_DIR_EITHER, - .mask_separate = BIT(IIO_EV_INFO_HYSTERESIS), - }, -}; - #define AD7091R_CHANNEL(idx, bits, ev, num_ev) { \ .type = IIO_VOLTAGE, \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ @@ -44,10 +24,10 @@ static const struct iio_event_spec ad7091r5_events[] = { .scan_type.realbits = bits, \ } static const struct iio_chan_spec ad7091r5_channels_irq[] = { - AD7091R_CHANNEL(0, 12, ad7091r5_events, ARRAY_SIZE(ad7091r5_events)), - AD7091R_CHANNEL(1, 12, ad7091r5_events, ARRAY_SIZE(ad7091r5_events)), - AD7091R_CHANNEL(2, 12, ad7091r5_events, ARRAY_SIZE(ad7091r5_events)), - AD7091R_CHANNEL(3, 12, ad7091r5_events, ARRAY_SIZE(ad7091r5_events)), + AD7091R_CHANNEL(0, 12, ad7091r_events, ARRAY_SIZE(ad7091r_events)), + AD7091R_CHANNEL(1, 12, ad7091r_events, ARRAY_SIZE(ad7091r_events)), + AD7091R_CHANNEL(2, 12, ad7091r_events, ARRAY_SIZE(ad7091r_events)), + AD7091R_CHANNEL(3, 12, ad7091r_events, ARRAY_SIZE(ad7091r_events)), }; static const struct iio_chan_spec ad7091r5_channels_noirq[] = { @@ -111,3 +91,4 @@ module_i2c_driver(ad7091r5_driver); MODULE_AUTHOR("Beniamin Bia "); MODULE_DESCRIPTION("Analog Devices AD7091R5 multi-channel ADC driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(IIO_AD7091R); diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c index 19a45dd437967e882a1a441478df4ba1ee6c5870..6b9627bfebb0ad5a96daf3e0ddf04d2b9bb660e5 100644 --- a/drivers/iio/adc/ad9467.c +++ b/drivers/iio/adc/ad9467.c @@ -119,9 +119,9 @@ struct ad9467_state { struct spi_device *spi; struct clk *clk; unsigned int output_mode; + unsigned int (*scales)[2]; struct gpio_desc *pwrdown_gpio; - struct gpio_desc *reset_gpio; }; static int ad9467_spi_read(struct spi_device *spi, unsigned int reg) @@ -163,9 +163,10 @@ static int ad9467_reg_access(struct adi_axi_adc_conv *conv, unsigned int reg, if (readval == NULL) { ret = ad9467_spi_write(spi, reg, writeval); - ad9467_spi_write(spi, AN877_ADC_REG_TRANSFER, - AN877_ADC_TRANSFER_SYNC); - return ret; + if (ret) + return ret; + return ad9467_spi_write(spi, AN877_ADC_REG_TRANSFER, + AN877_ADC_TRANSFER_SYNC); } ret = ad9467_spi_read(spi, reg); @@ -212,6 +213,7 @@ static void __ad9467_get_scale(struct adi_axi_adc_conv *conv, int index, .channel = _chan, \ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | \ BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SCALE), \ .scan_index = _si, \ .scan_type = { \ .sign = _sign, \ @@ -273,10 +275,13 @@ static int ad9467_get_scale(struct adi_axi_adc_conv *conv, int *val, int *val2) const struct ad9467_chip_info *info1 = to_ad9467_chip_info(info); struct ad9467_state *st = adi_axi_adc_conv_priv(conv); unsigned int i, vref_val; + int ret; - vref_val = ad9467_spi_read(st->spi, AN877_ADC_REG_VREF); + ret = ad9467_spi_read(st->spi, AN877_ADC_REG_VREF); + if (ret < 0) + return ret; - vref_val &= info1->vref_mask; + vref_val = ret & info1->vref_mask; for (i = 0; i < info->num_scales; i++) { if (vref_val == info->scale_table[i][1]) @@ -297,6 +302,7 @@ static int ad9467_set_scale(struct adi_axi_adc_conv *conv, int val, int val2) struct ad9467_state *st = adi_axi_adc_conv_priv(conv); unsigned int scale_val[2]; unsigned int i; + int ret; if (val != 0) return -EINVAL; @@ -306,11 +312,13 @@ static int ad9467_set_scale(struct adi_axi_adc_conv *conv, int val, int val2) if (scale_val[0] != val || scale_val[1] != val2) continue; - ad9467_spi_write(st->spi, AN877_ADC_REG_VREF, - info->scale_table[i][1]); - ad9467_spi_write(st->spi, AN877_ADC_REG_TRANSFER, - AN877_ADC_TRANSFER_SYNC); - return 0; + ret = ad9467_spi_write(st->spi, AN877_ADC_REG_VREF, + info->scale_table[i][1]); + if (ret < 0) + return ret; + + return ad9467_spi_write(st->spi, AN877_ADC_REG_TRANSFER, + AN877_ADC_TRANSFER_SYNC); } return -EINVAL; @@ -359,6 +367,26 @@ static int ad9467_write_raw(struct adi_axi_adc_conv *conv, } } +static int ad9467_read_avail(struct adi_axi_adc_conv *conv, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + const struct adi_axi_adc_chip_info *info = conv->chip_info; + struct ad9467_state *st = adi_axi_adc_conv_priv(conv); + + switch (mask) { + case IIO_CHAN_INFO_SCALE: + *vals = (const int *)st->scales; + *type = IIO_VAL_INT_PLUS_MICRO; + /* Values are stored in a 2D matrix */ + *length = info->num_scales * 2; + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + static int ad9467_outputmode_set(struct spi_device *spi, unsigned int mode) { int ret; @@ -371,6 +399,26 @@ static int ad9467_outputmode_set(struct spi_device *spi, unsigned int mode) AN877_ADC_TRANSFER_SYNC); } +static int ad9467_scale_fill(struct adi_axi_adc_conv *conv) +{ + const struct adi_axi_adc_chip_info *info = conv->chip_info; + struct ad9467_state *st = adi_axi_adc_conv_priv(conv); + unsigned int i, val1, val2; + + st->scales = devm_kmalloc_array(&st->spi->dev, info->num_scales, + sizeof(*st->scales), GFP_KERNEL); + if (!st->scales) + return -ENOMEM; + + for (i = 0; i < info->num_scales; i++) { + __ad9467_get_scale(conv, i, &val1, &val2); + st->scales[i][0] = val1; + st->scales[i][1] = val2; + } + + return 0; +} + static int ad9467_preenable_setup(struct adi_axi_adc_conv *conv) { struct ad9467_state *st = adi_axi_adc_conv_priv(conv); @@ -378,11 +426,19 @@ static int ad9467_preenable_setup(struct adi_axi_adc_conv *conv) return ad9467_outputmode_set(st->spi, st->output_mode); } -static void ad9467_clk_disable(void *data) +static int ad9467_reset(struct device *dev) { - struct ad9467_state *st = data; + struct gpio_desc *gpio; - clk_disable_unprepare(st->clk); + gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR_OR_NULL(gpio)) + return PTR_ERR_OR_ZERO(gpio); + + fsleep(1); + gpiod_set_value_cansleep(gpio, 0); + fsleep(10 * USEC_PER_MSEC); + + return 0; } static int ad9467_probe(struct spi_device *spi) @@ -404,40 +460,27 @@ static int ad9467_probe(struct spi_device *spi) st = adi_axi_adc_conv_priv(conv); st->spi = spi; - st->clk = devm_clk_get(&spi->dev, "adc-clk"); + st->clk = devm_clk_get_enabled(&spi->dev, "adc-clk"); if (IS_ERR(st->clk)) return PTR_ERR(st->clk); - ret = clk_prepare_enable(st->clk); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, ad9467_clk_disable, st); - if (ret) - return ret; - st->pwrdown_gpio = devm_gpiod_get_optional(&spi->dev, "powerdown", GPIOD_OUT_LOW); if (IS_ERR(st->pwrdown_gpio)) return PTR_ERR(st->pwrdown_gpio); - st->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset", - GPIOD_OUT_LOW); - if (IS_ERR(st->reset_gpio)) - return PTR_ERR(st->reset_gpio); - - if (st->reset_gpio) { - udelay(1); - ret = gpiod_direction_output(st->reset_gpio, 1); - if (ret) - return ret; - mdelay(10); - } + ret = ad9467_reset(&spi->dev); + if (ret) + return ret; spi_set_drvdata(spi, st); conv->chip_info = &info->axi_adc_info; + ret = ad9467_scale_fill(conv); + if (ret) + return ret; + id = ad9467_spi_read(spi, AN877_ADC_REG_CHIP_ID); if (id != conv->chip_info->id) { dev_err(&spi->dev, "Mismatch CHIP_ID, got 0x%X, expected 0x%X\n", @@ -448,6 +491,7 @@ static int ad9467_probe(struct spi_device *spi) conv->reg_access = ad9467_reg_access; conv->write_raw = ad9467_write_raw; conv->read_raw = ad9467_read_raw; + conv->read_avail = ad9467_read_avail; conv->preenable_setup = ad9467_preenable_setup; st->output_mode = info->default_output_mode | diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index cbe1011a2408aa54b9020304dd0175f6063cd3e6..d721fb1bcae7b27752d981c8bbaccf0dc139a20b 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -150,6 +150,20 @@ static int adi_axi_adc_write_raw(struct iio_dev *indio_dev, return conv->write_raw(conv, chan, val, val2, mask); } +static int adi_axi_adc_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + struct adi_axi_adc_state *st = iio_priv(indio_dev); + struct adi_axi_adc_conv *conv = &st->client->conv; + + if (!conv->read_avail) + return -EOPNOTSUPP; + + return conv->read_avail(conv, chan, vals, type, length, mask); +} + static int adi_axi_adc_update_scan_mode(struct iio_dev *indio_dev, const unsigned long *scan_mask) { @@ -238,69 +252,11 @@ struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev, } EXPORT_SYMBOL_GPL(devm_adi_axi_adc_conv_register); -static ssize_t in_voltage_scale_available_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct iio_dev *indio_dev = dev_to_iio_dev(dev); - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - size_t len = 0; - int i; - - for (i = 0; i < conv->chip_info->num_scales; i++) { - const unsigned int *s = conv->chip_info->scale_table[i]; - - len += scnprintf(buf + len, PAGE_SIZE - len, - "%u.%06u ", s[0], s[1]); - } - buf[len - 1] = '\n'; - - return len; -} - -static IIO_DEVICE_ATTR_RO(in_voltage_scale_available, 0); - -enum { - ADI_AXI_ATTR_SCALE_AVAIL, -}; - -#define ADI_AXI_ATTR(_en_, _file_) \ - [ADI_AXI_ATTR_##_en_] = &iio_dev_attr_##_file_.dev_attr.attr - -static struct attribute *adi_axi_adc_attributes[] = { - ADI_AXI_ATTR(SCALE_AVAIL, in_voltage_scale_available), - NULL -}; - -static umode_t axi_adc_attr_is_visible(struct kobject *kobj, - struct attribute *attr, int n) -{ - struct device *dev = kobj_to_dev(kobj); - struct iio_dev *indio_dev = dev_to_iio_dev(dev); - struct adi_axi_adc_state *st = iio_priv(indio_dev); - struct adi_axi_adc_conv *conv = &st->client->conv; - - switch (n) { - case ADI_AXI_ATTR_SCALE_AVAIL: - if (!conv->chip_info->num_scales) - return 0; - return attr->mode; - default: - return attr->mode; - } -} - -static const struct attribute_group adi_axi_adc_attribute_group = { - .attrs = adi_axi_adc_attributes, - .is_visible = axi_adc_attr_is_visible, -}; - static const struct iio_info adi_axi_adc_info = { .read_raw = &adi_axi_adc_read_raw, .write_raw = &adi_axi_adc_write_raw, - .attrs = &adi_axi_adc_attribute_group, .update_scan_mode = &adi_axi_adc_update_scan_mode, + .read_avail = &adi_axi_adc_read_avail, }; static const struct adi_axi_adc_core_info adi_axi_adc_10_0_a_info = { diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index 78ee9445f8019356cdd40706156a0311e6cb2e61..45a2d2b82b093a5f8271273b767ba66367e06815 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -322,12 +322,10 @@ struct iser_device { * * @mr: memory region * @sig_mr: signature memory region - * @mr_valid: is mr valid indicator */ struct iser_reg_resources { struct ib_mr *mr; struct ib_mr *sig_mr; - u8 mr_valid:1; }; /** diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index 27a6f75a9912ff2d005bbfc01719bc6646994f89..9ea88dd6a414e20e59458b3a7377950500f2902b 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c @@ -602,7 +602,10 @@ iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) return -EINVAL; } - desc->rsc.mr_valid = 0; + if (desc->sig_protected) + desc->rsc.sig_mr->need_inval = false; + else + desc->rsc.mr->need_inval = false; return 0; } diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c index d4e057fac219b8bcf4a561688d6412621ffb3703..519fea5ec3a1bf93e4cebbc8093061f4d6dd8f77 100644 --- a/drivers/infiniband/ulp/iser/iser_memory.c +++ b/drivers/infiniband/ulp/iser/iser_memory.c @@ -250,7 +250,7 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task, iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask); - if (rsc->mr_valid) + if (rsc->sig_mr->need_inval) iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); @@ -274,7 +274,7 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task, wr->access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE; - rsc->mr_valid = 1; + rsc->sig_mr->need_inval = true; sig_reg->sge.lkey = mr->lkey; sig_reg->rkey = mr->rkey; @@ -299,7 +299,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, struct ib_reg_wr *wr = &tx_desc->reg_wr; int n; - if (rsc->mr_valid) + if (rsc->mr->need_inval) iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); @@ -322,7 +322,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ; - rsc->mr_valid = 1; + rsc->mr->need_inval = true; reg->sge.lkey = mr->lkey; reg->rkey = mr->rkey; diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index 2bd18b0068934145de9440665eb80a438dc11a5a..b5127479860d8e82d51952549717951232de4c77 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -136,7 +136,6 @@ iser_create_fastreg_desc(struct iser_device *device, goto err_alloc_mr_integrity; } } - desc->rsc.mr_valid = 0; return desc; diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index 90dd3caa4612cc97e7372d7487bcdf510715848f..4912582c54dad0a8c36f08a314ca1cec43817ab6 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -782,9 +782,9 @@ static bool atkbd_is_portable_device(void) * not work. So in this case simply assume a keyboard is connected to avoid * confusing some laptop keyboards. * - * Skipping ATKBD_CMD_GETID ends up using a fake keyboard id. Using a fake id is - * ok in translated mode, only atkbd_select_set() checks atkbd->id and in - * translated mode that is a no-op. + * Skipping ATKBD_CMD_GETID ends up using a fake keyboard id. Using the standard + * 0xab83 id is ok in translated mode, only atkbd_select_set() checks atkbd->id + * and in translated mode that is a no-op. */ static bool atkbd_skip_getid(struct atkbd *atkbd) { @@ -802,6 +802,7 @@ static int atkbd_probe(struct atkbd *atkbd) { struct ps2dev *ps2dev = &atkbd->ps2dev; unsigned char param[2]; + bool skip_getid; /* * Some systems, where the bit-twiddling when testing the io-lines of the @@ -823,7 +824,8 @@ static int atkbd_probe(struct atkbd *atkbd) */ param[0] = param[1] = 0xa5; /* initialize with invalid values */ - if (atkbd_skip_getid(atkbd) || ps2_command(ps2dev, param, ATKBD_CMD_GETID)) { + skip_getid = atkbd_skip_getid(atkbd); + if (skip_getid || ps2_command(ps2dev, param, ATKBD_CMD_GETID)) { /* * If the get ID command was skipped or failed, we check if we can at least set @@ -833,7 +835,7 @@ static int atkbd_probe(struct atkbd *atkbd) param[0] = 0; if (ps2_command(ps2dev, param, ATKBD_CMD_SETLEDS)) return -1; - atkbd->id = 0xabba; + atkbd->id = skip_getid ? 0xab83 : 0xabba; return 0; } diff --git a/drivers/input/serio/i8042-loongsonio.h b/drivers/input/serio/i8042-loongsonio.h new file mode 100644 index 0000000000000000000000000000000000000000..2ea83b14f13d96797e71a20957a3d41661e3cf42 --- /dev/null +++ b/drivers/input/serio/i8042-loongsonio.h @@ -0,0 +1,330 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * i8042-loongsonio.h + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * Author: Jianmin Lv + * Huacai Chen + */ + +#ifndef _I8042_LOONGSONIO_H +#define _I8042_LOONGSONIO_H + +/* + * Names. + */ + +#define I8042_KBD_PHYS_DESC "isa0060/serio0" +#define I8042_AUX_PHYS_DESC "isa0060/serio1" +#define I8042_MUX_PHYS_DESC "isa0060/serio%d" + +/* + * IRQs. + */ +#define I8042_MAP_IRQ(x) (x) + +#define I8042_KBD_IRQ i8042_kbd_irq +#define I8042_AUX_IRQ i8042_aux_irq + +static int i8042_kbd_irq; +static int i8042_aux_irq; + +/* + * Register numbers. + */ + +#define I8042_COMMAND_REG i8042_command_reg +#define I8042_STATUS_REG i8042_command_reg +#define I8042_DATA_REG i8042_data_reg + +static int i8042_command_reg = 0x64; +static int i8042_data_reg = 0x60; + + +static inline int i8042_read_data(void) +{ + return inb(I8042_DATA_REG); +} + +static inline int i8042_read_status(void) +{ + return inb(I8042_STATUS_REG); +} + +static inline void i8042_write_data(int val) +{ + outb(val, I8042_DATA_REG); +} + +static inline void i8042_write_command(int val) +{ + outb(val, I8042_COMMAND_REG); +} + +#ifdef CONFIG_PNP +#include + +static bool i8042_pnp_kbd_registered; +static unsigned int i8042_pnp_kbd_devices; +static bool i8042_pnp_aux_registered; +static unsigned int i8042_pnp_aux_devices; + +static int i8042_pnp_command_reg; +static int i8042_pnp_data_reg; +static int i8042_pnp_kbd_irq; +static int i8042_pnp_aux_irq; + +static char i8042_pnp_kbd_name[32]; +static char i8042_pnp_aux_name[32]; + +static void i8042_pnp_id_to_string(struct pnp_id *id, char *dst, int dst_size) +{ + strlcpy(dst, "PNP:", dst_size); + + while (id) { + strlcat(dst, " ", dst_size); + strlcat(dst, id->id, dst_size); + id = id->next; + } +} + +static int i8042_pnp_kbd_probe(struct pnp_dev *dev, + const struct pnp_device_id *did) +{ + if (pnp_port_valid(dev, 0) && pnp_port_len(dev, 0) == 1) + i8042_pnp_data_reg = pnp_port_start(dev, 0); + + if (pnp_port_valid(dev, 1) && pnp_port_len(dev, 1) == 1) + i8042_pnp_command_reg = pnp_port_start(dev, 1); + + if (pnp_irq_valid(dev, 0)) + i8042_pnp_kbd_irq = pnp_irq(dev, 0); + + strlcpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name)); + if (strlen(pnp_dev_name(dev))) { + strlcat(i8042_pnp_kbd_name, ":", sizeof(i8042_pnp_kbd_name)); + strlcat(i8042_pnp_kbd_name, pnp_dev_name(dev), + sizeof(i8042_pnp_kbd_name)); + } + i8042_pnp_id_to_string(dev->id, i8042_kbd_firmware_id, + sizeof(i8042_kbd_firmware_id)); + + /* Keyboard ports are always supposed to be wakeup-enabled */ + device_set_wakeup_enable(&dev->dev, true); + + i8042_pnp_kbd_devices++; + return 0; +} + +static int i8042_pnp_aux_probe(struct pnp_dev *dev, + const struct pnp_device_id *did) +{ + if (pnp_port_valid(dev, 0) && pnp_port_len(dev, 0) == 1) + i8042_pnp_data_reg = pnp_port_start(dev, 0); + + if (pnp_port_valid(dev, 1) && pnp_port_len(dev, 1) == 1) + i8042_pnp_command_reg = pnp_port_start(dev, 1); + + if (pnp_irq_valid(dev, 0)) + i8042_pnp_aux_irq = pnp_irq(dev, 0); + + strlcpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name)); + if (strlen(pnp_dev_name(dev))) { + strlcat(i8042_pnp_aux_name, ":", sizeof(i8042_pnp_aux_name)); + strlcat(i8042_pnp_aux_name, pnp_dev_name(dev), + sizeof(i8042_pnp_aux_name)); + } + i8042_pnp_id_to_string(dev->id, i8042_aux_firmware_id, + sizeof(i8042_aux_firmware_id)); + + i8042_pnp_aux_devices++; + return 0; +} + +static const struct pnp_device_id pnp_kbd_devids[] = { + { .id = "PNP0300", .driver_data = 0 }, + { .id = "PNP0301", .driver_data = 0 }, + { .id = "PNP0302", .driver_data = 0 }, + { .id = "PNP0303", .driver_data = 0 }, + { .id = "PNP0304", .driver_data = 0 }, + { .id = "PNP0305", .driver_data = 0 }, + { .id = "PNP0306", .driver_data = 0 }, + { .id = "PNP0309", .driver_data = 0 }, + { .id = "PNP030a", .driver_data = 0 }, + { .id = "PNP030b", .driver_data = 0 }, + { .id = "PNP0320", .driver_data = 0 }, + { .id = "PNP0343", .driver_data = 0 }, + { .id = "PNP0344", .driver_data = 0 }, + { .id = "PNP0345", .driver_data = 0 }, + { .id = "CPQA0D7", .driver_data = 0 }, + { .id = "", }, +}; +MODULE_DEVICE_TABLE(pnp, pnp_kbd_devids); + +static struct pnp_driver i8042_pnp_kbd_driver = { + .name = "i8042 kbd", + .id_table = pnp_kbd_devids, + .probe = i8042_pnp_kbd_probe, + .driver = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, + .suppress_bind_attrs = true, + }, +}; + +static const struct pnp_device_id pnp_aux_devids[] = { + { .id = "AUI0200", .driver_data = 0 }, + { .id = "FJC6000", .driver_data = 0 }, + { .id = "FJC6001", .driver_data = 0 }, + { .id = "PNP0f03", .driver_data = 0 }, + { .id = "PNP0f0b", .driver_data = 0 }, + { .id = "PNP0f0e", .driver_data = 0 }, + { .id = "PNP0f12", .driver_data = 0 }, + { .id = "PNP0f13", .driver_data = 0 }, + { .id = "PNP0f19", .driver_data = 0 }, + { .id = "PNP0f1c", .driver_data = 0 }, + { .id = "SYN0801", .driver_data = 0 }, + { .id = "", }, +}; +MODULE_DEVICE_TABLE(pnp, pnp_aux_devids); + +static struct pnp_driver i8042_pnp_aux_driver = { + .name = "i8042 aux", + .id_table = pnp_aux_devids, + .probe = i8042_pnp_aux_probe, + .driver = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, + .suppress_bind_attrs = true, + }, +}; + +static void i8042_pnp_exit(void) +{ + if (i8042_pnp_kbd_registered) { + i8042_pnp_kbd_registered = false; + pnp_unregister_driver(&i8042_pnp_kbd_driver); + } + + if (i8042_pnp_aux_registered) { + i8042_pnp_aux_registered = false; + pnp_unregister_driver(&i8042_pnp_aux_driver); + } +} +#ifdef CONFIG_ACPI +#include +#endif +static int __init i8042_pnp_init(void) +{ + char kbd_irq_str[4] = { 0 }, aux_irq_str[4] = { 0 }; + bool pnp_data_busted = false; + int err; + + if (i8042_nopnp) { + pr_info("PNP detection disabled\n"); + return 0; + } + + err = pnp_register_driver(&i8042_pnp_kbd_driver); + if (!err) + i8042_pnp_kbd_registered = true; + + err = pnp_register_driver(&i8042_pnp_aux_driver); + if (!err) + i8042_pnp_aux_registered = true; + + if (!i8042_pnp_kbd_devices && !i8042_pnp_aux_devices) { + i8042_pnp_exit(); + pr_info("PNP: No PS/2 controller found.\n"); +#ifdef CONFIG_ACPI + if (acpi_disabled == 0) + return -ENODEV; +#endif + pr_info("Probing ports directly.\n"); + return 0; + } + + if (i8042_pnp_kbd_devices) + snprintf(kbd_irq_str, sizeof(kbd_irq_str), + "%d", i8042_pnp_kbd_irq); + if (i8042_pnp_aux_devices) + snprintf(aux_irq_str, sizeof(aux_irq_str), + "%d", i8042_pnp_aux_irq); + + pr_info("PNP: PS/2 Controller [%s%s%s] at %#x,%#x irq %s%s%s\n", + i8042_pnp_kbd_name, + (i8042_pnp_kbd_devices && i8042_pnp_aux_devices) ? "," : "", + i8042_pnp_aux_name, + i8042_pnp_data_reg, i8042_pnp_command_reg, + kbd_irq_str, + (i8042_pnp_kbd_devices && i8042_pnp_aux_devices) ? "," : "", + aux_irq_str); + + if (((i8042_pnp_data_reg & ~0xf) == (i8042_data_reg & ~0xf) && + i8042_pnp_data_reg != i8042_data_reg) || + !i8042_pnp_data_reg) { + pr_warn("PNP: PS/2 controller has invalid data port %#x; using default %#x\n", + i8042_pnp_data_reg, i8042_data_reg); + i8042_pnp_data_reg = i8042_data_reg; + pnp_data_busted = true; + } + + if (((i8042_pnp_command_reg & ~0xf) == (i8042_command_reg & ~0xf) && + i8042_pnp_command_reg != i8042_command_reg) || + !i8042_pnp_command_reg) { + pr_warn("PNP: PS/2 controller has invalid command port %#x; using default %#x\n", + i8042_pnp_command_reg, i8042_command_reg); + i8042_pnp_command_reg = i8042_command_reg; + pnp_data_busted = true; + } + + if (!i8042_nokbd && !i8042_pnp_kbd_irq) { + pr_warn("PNP: PS/2 controller doesn't have KBD irq; using default %d\n", + i8042_kbd_irq); + i8042_pnp_kbd_irq = i8042_kbd_irq; + pnp_data_busted = true; + } + + if (!i8042_noaux && !i8042_pnp_aux_irq) { + if (!pnp_data_busted && i8042_pnp_kbd_irq) { + pr_warn("PNP: PS/2 appears to have AUX port disabled, " + "if this is incorrect please boot with i8042.nopnp\n"); + i8042_noaux = true; + } else { + pr_warn("PNP: PS/2 controller doesn't have AUX irq; using default %d\n", + i8042_aux_irq); + i8042_pnp_aux_irq = i8042_aux_irq; + } + } + + i8042_data_reg = i8042_pnp_data_reg; + i8042_command_reg = i8042_pnp_command_reg; + i8042_kbd_irq = i8042_pnp_kbd_irq; + i8042_aux_irq = i8042_pnp_aux_irq; + + return 0; +} + +#else /* !CONFIG_PNP */ +static inline int i8042_pnp_init(void) { return 0; } +static inline void i8042_pnp_exit(void) { } +#endif /* CONFIG_PNP */ + +static int __init i8042_platform_init(void) +{ + int retval; + + i8042_kbd_irq = I8042_MAP_IRQ(1); + i8042_aux_irq = I8042_MAP_IRQ(12); + + retval = i8042_pnp_init(); + if (retval) + return retval; + + return retval; +} + +static inline void i8042_platform_exit(void) +{ + i8042_pnp_exit(); +} + +#endif /* _I8042_LOONGSONIO_H */ diff --git a/drivers/input/serio/i8042.h b/drivers/input/serio/i8042.h index bf2592fa9a783d17d32e4456b6e454738e5e42d3..6ce5cc007b0f30ce7fbac30b61cebd92ee97158c 100644 --- a/drivers/input/serio/i8042.h +++ b/drivers/input/serio/i8042.h @@ -19,6 +19,8 @@ #include "i8042-snirm.h" #elif defined(CONFIG_SPARC) #include "i8042-sparcio.h" +#elif defined(CONFIG_MACH_LOONGSON64) +#include "i8042-loongsonio.h" #elif defined(CONFIG_X86) || defined(CONFIG_IA64) #include "i8042-acpipnpio.h" #else diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 04878caf6da49a0243e71f003e768bf95675cee0..7dfd23262927c4e60858904d06b32a58539808bd 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -131,6 +131,7 @@ config MSM_IOMMU source "drivers/iommu/amd/Kconfig" source "drivers/iommu/intel/Kconfig" +source "drivers/iommu/loongson/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 11f1771104f3a1f9954b3baa3a886a346a00f454..8059c094910d957684661478226ee313ced9a28f 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += amd/ intel/ arm/ +obj-y += amd/ intel/ arm/ loongson/ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 1598a1ddbf694763945733519ff73d01bd4c0f84..a5164d5cb6a35678a3a5df11aa2d0a1fa05fed0a 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -21,6 +21,7 @@ static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu) static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,adreno" }, + { .compatible = "qcom,adreno-gmu" }, { .compatible = "qcom,mdp4" }, { .compatible = "qcom,mdss" }, { .compatible = "qcom,sc7180-mdss" }, diff --git a/drivers/iommu/loongson/Kconfig b/drivers/iommu/loongson/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..7b5d1b947c03d7f622c5bc894d8f2a5402adf293 --- /dev/null +++ b/drivers/iommu/loongson/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Loongson IOMMU support +config LOONGSON_IOMMU + bool "Loongson IOMMU support" + select IOMMU_API + select IOMMU_DMA + select IOMMU_DEFAULT_PASSTHROUGH + depends on LOONGARCH + help + With this option you can enable support for Loongson IOMMU hardware in + your system. An IOMMU is a hardware component which provides remapping + of DMA memory accesses from devices. With an IOMMU you can isolate the + DMA memory of different devices and protect the system from misbehaving + device drivers or hardware. diff --git a/drivers/iommu/loongson/Makefile b/drivers/iommu/loongson/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..99e38e56ebc27d38739d4bafe9fa1d2701e54aa4 --- /dev/null +++ b/drivers/iommu/loongson/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_LOONGSON_IOMMU) += iommu.o diff --git a/drivers/iommu/loongson/iommu.c b/drivers/iommu/loongson/iommu.c new file mode 100644 index 0000000000000000000000000000000000000000..9052b7b308272c4b72d2fa8d98a29512b9e454ae --- /dev/null +++ b/drivers/iommu/loongson/iommu.c @@ -0,0 +1,2039 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Loongson IOMMU Driver + * + * Copyright (C) 2020-2021 Loongson Technology Ltd. + * Author: Lv Chen + * Wang Yang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iommu.h" + +#define LOOP_TIMEOUT 100000 +#define IOVA_START (SZ_256M) +#define IOVA_END0 (SZ_2G + SZ_256M) + +#define IVRS_HEADER_LENGTH 48 +#define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 +#define IVHD_DEV_ALL 0x01 +#define IVHD_DEV_SELECT 0x02 +#define IVHD_DEV_SELECT_RANGE_START 0x03 +#define IVHD_DEV_RANGE_END 0x04 +#define IVHD_DEV_ALIAS 0x42 +#define IVHD_DEV_EXT_SELECT 0x46 +#define IVHD_DEV_ACPI_HID 0xf0 + +#define IVHD_HEAD_TYPE10 0x10 +#define IVHD_HEAD_TYPE11 0x11 +#define IVHD_HEAD_TYPE40 0x40 + +#define MAX_BDF_NUM 0xffff + +#define RLOOKUP_TABLE_ENTRY_SIZE (sizeof(void *)) + +/* + * structure describing one IOMMU in the ACPI table. Typically followed by one + * or more ivhd_entrys. + */ +struct ivhd_header { + u8 type; + u8 flags; + u16 length; + u16 devid; + u16 cap_ptr; + u64 mmio_phys; + u16 pci_seg; + u16 info; + u32 efr_attr; + + /* Following only valid on IVHD type 11h and 40h */ + u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ + u64 res; +} __attribute__((packed)); + +/* + * A device entry describing which devices a specific IOMMU translates and + * which requestor ids they use. + */ +struct ivhd_entry { + u8 type; + u16 devid; + u8 flags; + u32 ext; + u32 hidh; + u64 cid; + u8 uidf; + u8 uidl; + u8 uid; +} __attribute__((packed)); + +LIST_HEAD(loongson_iommu_list); /* list of all IOMMUs in the system */ +LIST_HEAD(loongson_rlookup_iommu_list); + +static u32 rlookup_table_size; /* size if the rlookup table */ +static int loongson_iommu_target_ivhd_type; +u16 loongson_iommu_last_bdf; /* largest PCI device id we have to handle */ + +int loongson_iommu_disable; +static struct iommu_ops loongson_iommu_ops; + +static void iommu_write_regl(loongson_iommu *iommu, unsigned long off, u32 val) +{ + *(u32 *)(iommu->membase + off) = val; + iob(); +} + +static u32 iommu_read_regl(loongson_iommu *iommu, unsigned long off) +{ + u32 val; + + val = *(u32 *)(iommu->membase + off); + iob(); + return val; +} + +static void iommu_translate_disable(loongson_iommu *iommu) +{ + u32 val; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + val = iommu_read_regl(iommu, LA_IOMMU_EIVDB); + + /* Disable */ + val &= ~(1 << 31); + iommu_write_regl(iommu, LA_IOMMU_EIVDB, val); + + /* Write cmd */ + val = iommu_read_regl(iommu, LA_IOMMU_CMD); + val &= 0xfffffffc; + iommu_write_regl(iommu, LA_IOMMU_CMD, val); +} + +static void iommu_translate_enable(loongson_iommu *iommu) +{ + u32 val = 0; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + val = iommu_read_regl(iommu, LA_IOMMU_EIVDB); + + /* Enable */ + val |= (1 << 31); + iommu_write_regl(iommu, LA_IOMMU_EIVDB, val); + + /* Write cmd */ + val = iommu_read_regl(iommu, LA_IOMMU_CMD); + val &= 0xfffffffc; + iommu_write_regl(iommu, LA_IOMMU_CMD, val); +} + +static bool loongson_iommu_capable(enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return true; + default: + return false; + } +} + +static dom_info *to_dom_info(struct iommu_domain *dom) +{ + return container_of(dom, dom_info, domain); +} + +/* + * Check whether the system has a priv. + * If yes, it returns 1 and if not, it returns 0 + */ +static int has_dom(loongson_iommu *iommu) +{ + spin_lock(&iommu->dom_info_lock); + while (!list_empty(&iommu->dom_list)) { + spin_unlock(&iommu->dom_info_lock); + return 1; + } + spin_unlock(&iommu->dom_info_lock); + + return 0; +} + +static int update_dev_table(struct loongson_iommu_dev_data *dev_data, int flag) +{ + u32 val = 0; + int index; + unsigned short bdf; + loongson_iommu *iommu; + u16 domain_id; + + if (dev_data == NULL) { + pr_err("%s dev_data is NULL", __func__); + return 0; + } + + if (dev_data->iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return 0; + } + + if (dev_data->iommu_entry == NULL) { + pr_err("%s iommu_entry is NULL", __func__); + return 0; + } + + iommu = dev_data->iommu; + domain_id = dev_data->iommu_entry->id; + bdf = dev_data->bdf; + + /* Set device table */ + if (flag) { + index = find_first_zero_bit(iommu->devtable_bitmap, MAX_ATTACHED_DEV_ID); + if (index < MAX_ATTACHED_DEV_ID) { + __set_bit(index, iommu->devtable_bitmap); + dev_data->index = index; + } else { + pr_err("%s get id from dev table failed\n", __func__); + return 0; + } + + pr_info("%s bdf %x domain_id %d index %x" + " iommu segment %d flag %x\n", + __func__, bdf, domain_id, index, + iommu->segment, flag); + + val = bdf & 0xffff; + val |= ((domain_id & 0xf) << 16); /* domain id */ + val |= ((index & 0xf) << 24); /* index */ + val |= (0x1 << 20); /* valid */ + val |= (0x1 << 31); /* enable */ + iommu_write_regl(iommu, LA_IOMMU_EIVDB, val); + + val = iommu_read_regl(iommu, LA_IOMMU_CMD); + val &= 0xfffffffc; + iommu_write_regl(iommu, LA_IOMMU_CMD, val); + } else { + /* Flush device table */ + index = dev_data->index; + pr_info("%s bdf %x domain_id %d index %x" + " iommu segment %d flag %x\n", + __func__, bdf, domain_id, index, + iommu->segment, flag); + + val = iommu_read_regl(iommu, LA_IOMMU_EIVDB); + val &= ~(0x7fffffff); + val |= ((index & 0xf) << 24); /* index */ + iommu_write_regl(iommu, LA_IOMMU_EIVDB, val); + + val = iommu_read_regl(iommu, LA_IOMMU_CMD); + val &= 0xfffffffc; + iommu_write_regl(iommu, LA_IOMMU_CMD, val); + + if (index < MAX_ATTACHED_DEV_ID) + __clear_bit(index, iommu->devtable_bitmap); + } + + return 0; +} + +static void flush_iotlb(loongson_iommu *iommu) +{ + u32 val, cmd; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + val = iommu_read_regl(iommu, LA_IOMMU_VBTC); + val &= ~0x1f; + + /* Flush all tlb */ + val |= 0x5; + iommu_write_regl(iommu, LA_IOMMU_VBTC, val); + + cmd = iommu_read_regl(iommu, LA_IOMMU_CMD); + cmd &= 0xfffffffc; + iommu_write_regl(iommu, LA_IOMMU_CMD, cmd); +} + +static int flush_pgtable_is_busy(loongson_iommu *iommu) +{ + u32 val; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return 0; + } + + val = iommu_read_regl(iommu, LA_IOMMU_VBTC); + + return val & IOMMU_PGTABLE_BUSY; +} + +static int __iommu_flush_iotlb_all(loongson_iommu *iommu) +{ + u32 retry = 0; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return 0; + } + + flush_iotlb(iommu); + while (flush_pgtable_is_busy(iommu)) { + if (retry == LOOP_TIMEOUT) { + pr_err("Loongson-IOMMU: iotlb flush busy\n"); + return -EIO; + } + retry++; + udelay(1); + } + iommu_translate_enable(iommu); + + return 0; +} + +static void priv_flush_iotlb_pde(loongson_iommu *iommu) +{ + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + __iommu_flush_iotlb_all(iommu); +} + +static void do_attach(iommu_info *info, struct loongson_iommu_dev_data *dev_data) +{ + if (!dev_data->count) + return; + + dev_data->iommu_entry = info; + + spin_lock(&info->devlock); + list_add(&dev_data->list, &info->dev_list); + info->dev_cnt += 1; + spin_unlock(&info->devlock); + + update_dev_table(dev_data, 1); + if (info->dev_cnt > 0) + priv_flush_iotlb_pde(dev_data->iommu); +} + +static void do_detach(struct loongson_iommu_dev_data *dev_data) +{ + iommu_info *iommu_entry = NULL; + + if (dev_data == NULL) { + pr_err("%s dev_data is NULL", __func__); + return; + } + + if (dev_data->count) + return; + + iommu_entry = dev_data->iommu_entry; + if (iommu_entry == NULL) { + pr_err("%s iommu_entry is NULL", __func__); + return; + } + + list_del(&dev_data->list); + iommu_entry->dev_cnt -= 1; + + update_dev_table(dev_data, 0); + dev_data->iommu_entry = NULL; +} + +static void cleanup_iommu_entry(iommu_info *iommu_entry) +{ + struct loongson_iommu_dev_data *dev_data = NULL; + + spin_lock(&iommu_entry->devlock); + while (!list_empty(&iommu_entry->dev_list)) { + dev_data = list_first_entry(&iommu_entry->dev_list, + struct loongson_iommu_dev_data, list); + do_detach(dev_data); + } + + spin_unlock(&iommu_entry->devlock); +} + +static int domain_id_alloc(loongson_iommu *iommu) +{ + int id = -1; + + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return id; + } + + spin_lock(&iommu->domain_bitmap_lock); + id = find_first_zero_bit(iommu->domain_bitmap, MAX_DOMAIN_ID); + if (id < MAX_DOMAIN_ID) + __set_bit(id, iommu->domain_bitmap); + else + pr_err("Loongson-IOMMU: Alloc domain id over max domain id\n"); + + spin_unlock(&iommu->domain_bitmap_lock); + + return id; +} + +static void domain_id_free(loongson_iommu *iommu, int id) +{ + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + spin_lock(&iommu->domain_bitmap_lock); + if ((id >= 0) && (id < MAX_DOMAIN_ID)) + __clear_bit(id, iommu->domain_bitmap); + + spin_unlock(&iommu->domain_bitmap_lock); +} + +/* + * This function adds a private domain to the global domain list + */ +static void add_domain_to_list(loongson_iommu *iommu, dom_info *priv) +{ + spin_lock(&iommu->dom_info_lock); + list_add(&priv->list, &iommu->dom_list); + spin_unlock(&iommu->dom_info_lock); +} + +static void del_domain_from_list(loongson_iommu *iommu, dom_info *priv) +{ + spin_lock(&iommu->dom_info_lock); + list_del(&priv->list); + spin_unlock(&iommu->dom_info_lock); +} + +static spt_entry *iommu_zalloc_page(loongson_iommu *iommu) +{ + int index; + void *addr; + spt_entry *shd_entry; + + spin_lock(&iommu->pgtable_bitmap_lock); + index = find_first_zero_bit(iommu->pgtable_bitmap, iommu->maxpages); + if (index < iommu->maxpages) + __set_bit(index, iommu->pgtable_bitmap); + spin_unlock(&iommu->pgtable_bitmap_lock); + + shd_entry = NULL; + if (index < iommu->maxpages) { + shd_entry = kmalloc(sizeof(*shd_entry), GFP_KERNEL); + if (!shd_entry) { + pr_err("%s alloc memory for shadow page entry failed\n", __func__); + goto fail; + } + + shd_entry->shadow_ptable = (unsigned long *)get_zeroed_page(GFP_KERNEL); + if (!shd_entry->shadow_ptable) { + pr_err("Loongson-IOMMU: get zeroed page err\n"); + kfree(shd_entry); + goto fail; + } + + addr = iommu->pgtbase + index * IOMMU_PAGE_SIZE; + memset(addr, 0x0, IOMMU_PAGE_SIZE); + shd_entry->index = index; + shd_entry->gmem_ptable = addr; + } + + return shd_entry; +fail: + spin_lock(&iommu->pgtable_bitmap_lock); + __clear_bit(index, iommu->pgtable_bitmap); + spin_unlock(&iommu->pgtable_bitmap_lock); + return NULL; +} + +static void iommu_free_page(loongson_iommu *iommu, spt_entry *shadw_entry) +{ + void *addr; + + if (shadw_entry->index < iommu->maxpages) { + addr = shadw_entry->gmem_ptable; + memset(addr, 0x0, IOMMU_PAGE_SIZE); + + spin_lock(&iommu->pgtable_bitmap_lock); + __clear_bit(shadw_entry->index, iommu->pgtable_bitmap); + spin_unlock(&iommu->pgtable_bitmap_lock); + + shadw_entry->index = -1; + free_page((unsigned long)shadw_entry->shadow_ptable); + shadw_entry->shadow_ptable = NULL; + shadw_entry->gmem_ptable = NULL; + kfree(shadw_entry); + } +} + +static void free_pagetable_one_level(iommu_info *iommu_entry, spt_entry *shd_entry, int level) +{ + int i; + unsigned long *psentry; + spt_entry *shd_entry_tmp; + loongson_iommu *iommu = iommu_entry->iommu; + + psentry = (unsigned long *)shd_entry; + if (level == IOMMU_PT_LEVEL1) { + if (iommu_pt_present(psentry) && (!iommu_pt_huge(psentry))) + iommu_free_page(iommu, shd_entry); + return; + } + + for (i = 0; i < IOMMU_PTRS_PER_LEVEL; i++) { + psentry = shd_entry->shadow_ptable + i; + if (!iommu_pt_present(psentry)) + continue; + + shd_entry_tmp = (spt_entry *)(*psentry); + free_pagetable_one_level(iommu_entry, shd_entry_tmp, level - 1); + } + + iommu_free_page(iommu, shd_entry); +} + +static void free_pagetable(iommu_info *iommu_entry) +{ + spt_entry *shd_entry; + loongson_iommu *iommu; + + iommu = iommu_entry->iommu; + shd_entry = iommu_entry->shadow_pgd; + free_pagetable_one_level(iommu_entry, shd_entry, IOMMU_LEVEL_MAX); + iommu_entry->shadow_pgd = NULL; +} + +static dom_info *dom_info_alloc(void) +{ + dom_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (info == NULL) { + pr_err("%s alloc memory for info failed\n", __func__); + return NULL; + } + + /* 0x10000000~0x8fffffff */ + info->mmio_pgd = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 6); + if (info->mmio_pgd == NULL) { + pr_err("%s alloc memory for virtio pgtable failed\n", __func__); + kfree(info); + return NULL; + } + + INIT_LIST_HEAD(&info->iommu_devlist); + spin_lock_init(&info->lock); + return info; +} + +static void dom_info_free(dom_info *info) +{ + /* 0x10000000~0x8fffffff */ + if (info->mmio_pgd) { + free_pages((unsigned long)info->mmio_pgd, 6); + info->mmio_pgd = NULL; + } + + kfree(info); +} + +static struct iommu_domain *loongson_iommu_domain_alloc(unsigned int type) +{ + dom_info *info; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + info = dom_info_alloc(); + if (info == NULL) + return NULL; + + info->domain.geometry.aperture_start = 0; + info->domain.geometry.aperture_end = ~0ULL; + info->domain.geometry.force_aperture = true; + break; + + default: + return NULL; + } + + return &info->domain; +} + +void domain_deattach_iommu(dom_info *priv, iommu_info *iommu_entry) +{ + loongson_iommu *iommu = NULL; + + if (priv == NULL) { + pr_err("%s priv is NULL", __func__); + return; + } + + if (iommu_entry == NULL) { + pr_err("%s iommu_entry is NULL", __func__); + return; + } + + if (iommu_entry->dev_cnt != 0) + return; + + iommu = iommu_entry->iommu; + if (iommu == NULL) { + pr_err("%s iommu is NULL", __func__); + return; + } + + domain_id_free(iommu_entry->iommu, iommu_entry->id); + + mutex_lock(&iommu->loongson_iommu_pgtlock); + free_pagetable(iommu_entry); + mutex_unlock(&iommu->loongson_iommu_pgtlock); + + spin_lock(&priv->lock); + list_del(&iommu_entry->list); + spin_unlock(&priv->lock); + + kfree(iommu_entry); + del_domain_from_list(iommu, priv); +} + +static void loongson_iommu_domain_free(struct iommu_domain *domain) +{ + + dom_info *priv; + loongson_iommu *iommu = NULL; + struct iommu_info *iommu_entry, *iommu_entry_temp; + + priv = to_dom_info(domain); + + spin_lock(&priv->lock); + list_for_each_entry_safe(iommu_entry, iommu_entry_temp, &priv->iommu_devlist, list) { + iommu = iommu_entry->iommu; + + if (iommu_entry->dev_cnt > 0) + cleanup_iommu_entry(iommu_entry); + + spin_unlock(&priv->lock); + domain_deattach_iommu(priv, iommu_entry); + spin_lock(&priv->lock); + + __iommu_flush_iotlb_all(iommu); + + if (!has_dom(iommu)) + iommu_translate_disable(iommu); + + } + spin_unlock(&priv->lock); + + dom_info_free(priv); +} + +struct loongson_iommu_rlookup_entry *lookup_rlooptable(int pcisegment) +{ + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + list_for_each_entry(rlookupentry, &loongson_rlookup_iommu_list, list) { + if (rlookupentry->pcisegment == pcisegment) + return rlookupentry; + } + + return NULL; +} + +loongson_iommu *find_iommu_by_dev(struct pci_dev *pdev) +{ + int pcisegment; + unsigned short devid; + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + loongson_iommu *iommu = NULL; + + devid = pdev->devfn & 0xff; + + pcisegment = pci_domain_nr(pdev->bus); + + rlookupentry = lookup_rlooptable(pcisegment); + if (rlookupentry == NULL) { + pr_info("%s find segment %d rlookupentry failed\n", __func__, + pcisegment); + return iommu; + } + + iommu = rlookupentry->loongson_iommu_rlookup_table[devid]; + + return iommu; +} + +static int iommu_init_device(struct device *dev) +{ + unsigned char busnum; + unsigned short bdf, devid; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_bus *bus = pdev->bus; + struct loongson_iommu_dev_data *dev_data; + loongson_iommu *iommu = NULL; + + bdf = pdev->devfn & 0xff; + busnum = bus->number; + if (busnum != 0) { + while (bus->parent->parent) + bus = bus->parent; + bdf = bus->self->devfn & 0xff; + } + + if (dev_iommu_priv_get(dev)) { + pr_info("Loongson-IOMMU: bdf:0x%x has added\n", bdf); + return 0; + } + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + devid = PCI_DEVID(bus->number, bdf); + dev_data->bdf = devid; + + pci_info(pdev, "%s devid %x bus %x\n", __func__, devid, busnum); + iommu = find_iommu_by_dev(pdev); + if (iommu == NULL) + pci_info(pdev, "%s find iommu failed by dev\n", __func__); + + /* The initial state is 0, and 1 is added only when attach dev */ + dev_data->count = 0; + dev_data->iommu = iommu; + + dev_iommu_priv_set(dev, dev_data); + + return 0; +} + +static struct iommu_device *loongson_iommu_probe_device(struct device *dev) +{ + int ret = 0; + + ret = iommu_init_device(dev); + if (ret < 0) + pr_err("Loongson-IOMMU: unable to alloc memory for dev_data\n"); + + return 0; +} + +static struct iommu_group *loongson_iommu_device_group(struct device *dev) +{ + struct iommu_group *group; + + /* + * We don't support devices sharing stream IDs other than PCI RID + * aliases, since the necessary ID-to-device lookup becomes rather + * impractical given a potential sparse 32-bit stream ID space. + */ + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else + group = generic_device_group(dev); + + return group; +} + +static void loongson_iommu_release_device(struct device *dev) +{ + struct loongson_iommu_dev_data *dev_data; + + dev_data = dev_iommu_priv_get(dev); + dev_iommu_priv_set(dev, NULL); + kfree(dev_data); +} + +iommu_info *get_first_iommu_entry(dom_info *priv) +{ + struct iommu_info *iommu_entry; + + if (priv == NULL) { + pr_err("%s priv is NULL", __func__); + return NULL; + } + + iommu_entry = list_first_entry_or_null(&priv->iommu_devlist, + struct iommu_info, list); + + return iommu_entry; +} + +iommu_info *get_iommu_entry(dom_info *priv, loongson_iommu *iommu) +{ + struct iommu_info *iommu_entry; + + spin_lock(&priv->lock); + list_for_each_entry(iommu_entry, &priv->iommu_devlist, list) { + if (iommu_entry->iommu == iommu) { + spin_unlock(&priv->lock); + return iommu_entry; + } + } + spin_unlock(&priv->lock); + + return NULL; +} + +iommu_info *domain_attach_iommu(dom_info *priv, loongson_iommu *iommu) +{ + unsigned long pgd_pa; + u32 dir_ctrl, pgd_lo, pgd_hi; + struct iommu_info *iommu_entry = NULL; + spt_entry *shd_entry = NULL; + + iommu_entry = get_iommu_entry(priv, iommu); + if (iommu_entry) + return iommu_entry; + + iommu_entry = kzalloc(sizeof(struct iommu_info), GFP_KERNEL); + if (iommu_entry == NULL) { + pr_info("%s alloc memory for iommu_entry failed\n", __func__); + return NULL; + } + + INIT_LIST_HEAD(&iommu_entry->dev_list); + iommu_entry->iommu = iommu; + iommu_entry->id = domain_id_alloc(iommu); + if (iommu_entry->id == -1) { + pr_info("%s alloc id for domain failed\n", __func__); + kfree(iommu_entry); + return NULL; + } + + shd_entry = iommu_zalloc_page(iommu); + if (!shd_entry) { + pr_info("%s alloc shadow page entry err\n", __func__); + domain_id_free(iommu, iommu_entry->id); + kfree(iommu_entry); + return NULL; + } + + iommu_entry->shadow_pgd = shd_entry; + dir_ctrl = (IOMMU_LEVEL_STRIDE << 26) | (IOMMU_LEVEL_SHIFT(2) << 20); + dir_ctrl |= (IOMMU_LEVEL_STRIDE << 16) | (IOMMU_LEVEL_SHIFT(1) << 10); + dir_ctrl |= (IOMMU_LEVEL_STRIDE << 6) | IOMMU_LEVEL_SHIFT(0); + pgd_pa = iommu_pgt_v2p(iommu, shd_entry->gmem_ptable); + pgd_hi = pgd_pa >> 32; + pgd_lo = pgd_pa & 0xffffffff; + iommu_write_regl(iommu, LA_IOMMU_DIR_CTRL(iommu_entry->id), dir_ctrl); + iommu_write_regl(iommu, LA_IOMMU_PGD_HI(iommu_entry->id), pgd_hi); + iommu_write_regl(iommu, LA_IOMMU_PGD_LO(iommu_entry->id), pgd_lo); + + spin_lock(&priv->lock); + list_add(&iommu_entry->list, &priv->iommu_devlist); + spin_unlock(&priv->lock); + + add_domain_to_list(iommu, priv); + pr_info("%s iommu_entry->iommu %lx id %x\n", __func__, + (unsigned long)iommu_entry->iommu, iommu_entry->id); + + return iommu_entry; +} + +static struct loongson_iommu_dev_data *iommu_get_devdata(dom_info *info, + loongson_iommu *iommu, unsigned long bdf) +{ + struct iommu_info *entry; + struct loongson_iommu_dev_data *dev_data; + + entry = get_iommu_entry(info, iommu); + if (!entry) + return NULL; + + /* Find from priv list */ + spin_lock(&entry->devlock); + list_for_each_entry(dev_data, &entry->dev_list, list) { + if (dev_data->bdf == bdf) { + spin_unlock(&entry->devlock); + return dev_data; + } + } + + spin_unlock(&entry->devlock); + return NULL; +} + +static int loongson_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + unsigned short bdf; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_bus *bus = pdev->bus; + unsigned char busnum = pdev->bus->number; + struct loongson_iommu_dev_data *dev_data; + dom_info *priv = to_dom_info(domain); + loongson_iommu *iommu; + iommu_info *iommu_entry = NULL; + + bdf = pdev->devfn & 0xff; + if (busnum != 0) { + while (bus->parent->parent) + bus = bus->parent; + bdf = bus->self->devfn & 0xff; + } + + dev_data = (struct loongson_iommu_dev_data *)dev_iommu_priv_get(dev); + if (dev_data == NULL) { + pci_info(pdev, "%s dev_data is Invalid\n", __func__); + return 0; + } + + iommu = dev_data->iommu; + if (iommu == NULL) { + pci_info(pdev, "%s iommu is Invalid\n", __func__); + return 0; + } + + pci_info(pdev, "%s busnum %x bdf %x priv %lx iommu %lx\n", __func__, + busnum, bdf, (unsigned long)priv, (unsigned long)iommu); + dev_data = iommu_get_devdata(priv, iommu, bdf); + if (dev_data) { + dev_data->count++; + pci_info(pdev, "Loongson-IOMMU: bdf 0x%x devfn %x has attached," + " count:0x%x\n", + bdf, pdev->devfn, dev_data->count); + return 0; + } else { + dev_data = (struct loongson_iommu_dev_data *)dev_iommu_priv_get(dev); + } + + iommu_entry = domain_attach_iommu(priv, iommu); + if (iommu_entry == NULL) { + pci_info(pdev, "domain attach iommu failed\n"); + return 0; + } + + dev_data->count++; + do_attach(iommu_entry, dev_data); + + return 0; +} + +static void loongson_iommu_detach_dev(struct iommu_domain *domain, + struct device *dev) +{ + unsigned short bdf; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_bus *bus = pdev->bus; + unsigned char busnum = pdev->bus->number; + struct loongson_iommu_dev_data *dev_data; + dom_info *priv = to_dom_info(domain); + loongson_iommu *iommu; + iommu_info *iommu_entry = NULL; + + bdf = pdev->devfn & 0xff; + if (busnum != 0) { + while (bus->parent->parent) + bus = bus->parent; + bdf = bus->self->devfn & 0xff; + } + + dev_data = (struct loongson_iommu_dev_data *)dev_iommu_priv_get(dev); + if (dev_data == NULL) { + pci_info(pdev, "%s dev_data is Invalid\n", __func__); + return; + } + + iommu = dev_data->iommu; + if (iommu == NULL) { + pci_info(pdev, "%s iommu is Invalid\n", __func__); + return; + } + + dev_data = iommu_get_devdata(priv, iommu, bdf); + if (dev_data == NULL) { + pci_info(pdev, "Loongson-IOMMU: bdf 0x%x devfn %x dev_data is NULL\n", + bdf, pdev->devfn & 0xff); + return; + } + + iommu = dev_data->iommu; + dev_data->count--; + iommu_entry = get_iommu_entry(priv, iommu); + if (iommu_entry == NULL) { + pci_info(pdev, "%s get iommu_entry failed\n", __func__); + return; + } + + spin_lock(&iommu_entry->devlock); + do_detach(dev_data); + spin_unlock(&iommu_entry->devlock); + + pci_info(pdev, "%s iommu devid %x sigment %x\n", __func__, + iommu->devid, iommu->segment); +} + +static unsigned long *iommu_get_spte(spt_entry *entry, unsigned long iova, int level) +{ + int i; + unsigned long *pte; + + if (level > (IOMMU_LEVEL_MAX - 1)) + return NULL; + + for (i = IOMMU_LEVEL_MAX - 1; i >= level; i--) { + pte = iommu_shadow_offset(entry, iova, i); + if (!iommu_pt_present(pte)) + break; + + if (iommu_pt_huge(pte)) + break; + + entry = (spt_entry *)(*pte); + } + + return pte; +} + +static int _iommu_alloc_ptable(loongson_iommu *iommu, + unsigned long *psentry, unsigned long *phwentry) +{ + unsigned long pte; + iommu_pte *new_phwentry; + spt_entry *new_shd_entry; + + if (!iommu_pt_present(psentry)) { + new_shd_entry = iommu_zalloc_page(iommu); + if (!new_shd_entry) { + pr_err("Loongson-IOMMU: new_shd_entry alloc err\n"); + return -ENOMEM; + } + /* fill shd_entry */ + *psentry = (unsigned long)new_shd_entry; + /* fill gmem phwentry */ + new_phwentry = (iommu_pte *)new_shd_entry->gmem_ptable; + pte = iommu_pgt_v2p(iommu, new_phwentry) & IOMMU_PAGE_MASK; + pte |= IOMMU_PTE_RW; + *phwentry = pte; + } + + return 0; +} + +static size_t iommu_ptw_map(loongson_iommu *iommu, spt_entry *shd_entry, + unsigned long start, unsigned long end, phys_addr_t pa, int level) +{ + int ret, huge; + unsigned long pte; + unsigned long next, old, step; + unsigned long *psentry, *phwentry; + + old = start; + psentry = iommu_shadow_offset(shd_entry, start, level); + phwentry = iommu_ptable_offset(shd_entry->gmem_ptable, start, level); + if (level == IOMMU_PT_LEVEL0) { + pa = pa & IOMMU_PAGE_MASK; + do { + pte = pa | IOMMU_PTE_RW; + *phwentry = pte; + *psentry = pte; + psentry++; + phwentry++; + start += IOMMU_PAGE_SIZE; + pa += IOMMU_PAGE_SIZE; + } while (start < end); + + return start - old; + } + + do { + next = iommu_ptable_end(start, end, level); + step = next - start; + + huge = 0; + if ((level == IOMMU_PT_LEVEL1) && (step == IOMMU_HPAGE_SIZE)) + if (!iommu_pt_present(psentry) || iommu_pt_huge(psentry)) + huge = 1; + + if (huge) { + pte = (pa & IOMMU_HPAGE_MASK) | IOMMU_PTE_RW | IOMMU_PTE_HP; + *phwentry = pte; + *psentry = pte; + } else { + ret = _iommu_alloc_ptable(iommu, psentry, phwentry); + if (ret != 0) + break; + iommu_ptw_map(iommu, (spt_entry *)*psentry, start, next, pa, level - 1); + } + + psentry++; + phwentry++; + pa += step; + start = next; + } while (start < end); + + return start - old; +} + +static int dev_map_page(iommu_info *iommu_entry, unsigned long start, + phys_addr_t pa, size_t size) +{ + int ret = 0; + spt_entry *entry; + phys_addr_t end; + size_t map_size; + loongson_iommu *iommu; + + end = start + size; + iommu = iommu_entry->iommu; + + mutex_lock(&iommu->loongson_iommu_pgtlock); + entry = iommu_entry->shadow_pgd; + map_size = iommu_ptw_map(iommu, entry, start, end, pa, IOMMU_LEVEL_MAX - 1); + if (map_size != size) + ret = -EFAULT; + + if (has_dom(iommu)) + __iommu_flush_iotlb_all(iommu); + mutex_unlock(&iommu->loongson_iommu_pgtlock); + + return ret; +} + +static size_t iommu_ptw_unmap(loongson_iommu *iommu, spt_entry *shd_entry, + unsigned long start, unsigned long end, int level) +{ + unsigned long next, old; + unsigned long *psentry, *phwentry; + + old = start; + psentry = iommu_shadow_offset(shd_entry, start, level); + phwentry = iommu_ptable_offset(shd_entry->gmem_ptable, start, level); + if (level == IOMMU_PT_LEVEL0) { + do { + *phwentry++ = 0; + *psentry++ = 0; + start += IOMMU_PAGE_SIZE; + } while (start < end); + } else { + do { + next = iommu_ptable_end(start, end, level); + if (!iommu_pt_present(psentry)) + continue; + + if (iommu_pt_huge(psentry)) { + if ((next - start) != IOMMU_HPAGE_SIZE) + pr_err("Map pte on hugepage not supported now\n"); + *phwentry = 0; + *psentry = 0; + } else + iommu_ptw_unmap(iommu, (spt_entry *)*psentry, start, next, level - 1); + } while (psentry++, phwentry++, start = next, start < end); + } + + return start - old; +} + +static int iommu_map_page(dom_info *priv, unsigned long start, + phys_addr_t pa, size_t size, int prot, gfp_t gfp) +{ + + unsigned long *pte; + int ret = 0; + iommu_info *iommu_entry = NULL; + + /* 0x10000000~0x8fffffff */ + if ((start >= IOVA_START) && (start < IOVA_END0)) { + start -= IOVA_START; + pte = (unsigned long *)priv->mmio_pgd; + while (size > 0) { + pte[start >> LA_VIRTIO_PAGE_SHIFT] = + pa & LA_VIRTIO_PAGE_MASK; + size -= IOMMU_PAGE_SIZE; + start += IOMMU_PAGE_SIZE; + pa += IOMMU_PAGE_SIZE; + } + return 0; + } + + spin_lock(&priv->lock); + list_for_each_entry(iommu_entry, &priv->iommu_devlist, list) { + ret |= dev_map_page(iommu_entry, start, pa, size); + } + spin_unlock(&priv->lock); + + return ret; +} + +static size_t iommu_unmap_page(iommu_info *iommu_entry, unsigned long start, size_t size) +{ + loongson_iommu *iommu; + spt_entry *entry; + size_t unmap_len; + unsigned long end; + + end = start + size; + iommu = iommu_entry->iommu; + mutex_lock(&iommu->loongson_iommu_pgtlock); + entry = iommu_entry->shadow_pgd; + unmap_len = iommu_ptw_unmap(iommu, entry, start, end, (IOMMU_LEVEL_MAX - 1)); + + if (has_dom(iommu)) + __iommu_flush_iotlb_all(iommu); + mutex_unlock(&iommu->loongson_iommu_pgtlock); + return unmap_len; +} + +static size_t domain_unmap_page(dom_info *info, unsigned long start, size_t size) +{ + unsigned long *pte; + size_t unmap_len = 0; + iommu_info *entry; + + /* 0x10000000~0x8fffffff */ + if ((start >= IOVA_START) && (start < IOVA_END0)) { + start -= IOVA_START; + pte = (unsigned long *)info->mmio_pgd; + while (size > 0) { + pte[start >> LA_VIRTIO_PAGE_SHIFT] = 0; + size -= 0x4000; + unmap_len += 0x4000; + start += 0x4000; + } + unmap_len += size; + + return unmap_len; + } + + spin_lock(&info->lock); + list_for_each_entry(entry, &info->iommu_devlist, list) + unmap_len = iommu_unmap_page(entry, start, size); + spin_unlock(&info->lock); + + return unmap_len; +} + +static int loongson_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t pa, size_t len, int prot, gfp_t gfp) +{ + dom_info *priv = to_dom_info(domain); + + return iommu_map_page(priv, iova, pa, len, prot, GFP_KERNEL); +} + +static size_t loongson_iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size, struct iommu_iotlb_gather *gather) +{ + dom_info *priv = to_dom_info(domain); + + return domain_unmap_page(priv, iova, size); +} + +static phys_addr_t loongson_iommu_iova_to_pa(struct iommu_domain *domain, + dma_addr_t iova) +{ + unsigned long pa, offset, tmpva, page_size, page_mask; + dom_info *priv = to_dom_info(domain); + unsigned long *psentry, *pte; + int ret = 0; + spt_entry *entry; + loongson_iommu *iommu; + iommu_info *iommu_entry = NULL; + + /* 0x10000000~0x8fffffff */ + if ((iova >= IOVA_START) && (iova < IOVA_END0)) { + tmpva = iova & LA_VIRTIO_PAGE_MASK; + pte = (unsigned long *)priv->mmio_pgd; + offset = iova & ((1ULL << LA_VIRTIO_PAGE_SHIFT) - 1); + pa = pte[(tmpva - IOVA_START) >> 14] + offset; + + return pa; + } + + iommu_entry = get_first_iommu_entry(priv); + if (iommu_entry == NULL) { + pr_err("%s iova:0x%llx iommu_entry is invalid\n", + __func__, iova); + ret = -EFAULT; + return ret; + } + + iommu = iommu_entry->iommu; + + mutex_lock(&iommu->loongson_iommu_pgtlock); + entry = iommu_entry->shadow_pgd; + psentry = iommu_get_spte(entry, iova, IOMMU_PT_LEVEL0); + mutex_unlock(&iommu->loongson_iommu_pgtlock); + + if (!psentry || !iommu_pt_present(psentry)) { + ret = -EFAULT; + pr_warn_once("Loongson-IOMMU: shadow pte is null or not present with iova %llx \n", iova); + return ret; + } + + if (iommu_pt_huge(psentry)) { + page_size = IOMMU_HPAGE_SIZE; + page_mask = IOMMU_HPAGE_MASK; + } else { + page_size = IOMMU_PAGE_SIZE; + page_mask = IOMMU_PAGE_MASK; + } + + pa = *psentry & page_mask; + pa |= (iova & (page_size - 1)); + + return (phys_addr_t)pa; +} + +static phys_addr_t loongson_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + phys_addr_t pa; + + pa = loongson_iommu_iova_to_pa(domain, iova); + + return pa; +} + +static void loongson_iommu_flush_iotlb_all(struct iommu_domain *domain) +{ + int ret; + dom_info *priv = to_dom_info(domain); + iommu_info *iommu_entry; + loongson_iommu *iommu; + + spin_lock(&priv->lock); + list_for_each_entry(iommu_entry, &priv->iommu_devlist, list) { + iommu = iommu_entry->iommu; + + ret = __iommu_flush_iotlb_all(iommu); + } + spin_unlock(&priv->lock); +} + +static void loongson_iommu_iotlb_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + loongson_iommu_flush_iotlb_all(domain); +} + +static struct iommu_ops loongson_iommu_ops = { + .capable = loongson_iommu_capable, + .domain_alloc = loongson_iommu_domain_alloc, + .domain_free = loongson_iommu_domain_free, + .attach_dev = loongson_iommu_attach_dev, + .detach_dev = loongson_iommu_detach_dev, + .map = loongson_iommu_map, + .unmap = loongson_iommu_unmap, + .iova_to_phys = loongson_iommu_iova_to_phys, + .probe_device = loongson_iommu_probe_device, + .release_device = loongson_iommu_release_device, + .device_group = loongson_iommu_device_group, + .pgsize_bitmap = LA_IOMMU_PGSIZE, + .flush_iotlb_all = loongson_iommu_flush_iotlb_all, + .iotlb_sync = loongson_iommu_iotlb_sync, +}; + +loongson_iommu *loongarch_get_iommu(struct pci_dev *pdev) +{ + int pcisegment; + unsigned short devid; + loongson_iommu *iommu = NULL; + + devid = pdev->devfn & 0xff; + pcisegment = pci_domain_nr(pdev->bus); + + list_for_each_entry(iommu, &loongson_iommu_list, list) { + if ((iommu->segment == pcisegment) && + (iommu->devid == devid)) { + return iommu; + } + } + + return NULL; +} + +static int loongson_iommu_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + int ret = 1; + int bitmap_sz = 0; + int tmp; + struct loongson_iommu *iommu = NULL; + resource_size_t base, size; + + iommu = loongarch_get_iommu(pdev); + if (iommu == NULL) { + pci_info(pdev, "%s can't find iommu\n", __func__); + return -ENODEV; + } + + base = pci_resource_start(pdev, 0); + size = pci_resource_len(pdev, 0); + if (!request_mem_region(base, size, "loongson_iommu")) { + pci_err(pdev, "can't reserve mmio registers\n"); + return -ENOMEM; + } + + iommu->membase = ioremap(base, size); + if (iommu->membase == NULL) { + pci_info(pdev, "%s iommu pci dev bar0 is NULL\n", __func__); + return ret; + } + + base = pci_resource_start(pdev, 2); + size = pci_resource_len(pdev, 2); + if (!request_mem_region(base, size, "loongson_iommu")) { + pci_err(pdev, "can't reserve mmio registers\n"); + return -ENOMEM; + } + iommu->pgtbase = ioremap(base, size); + if (iommu->pgtbase == NULL) + return -ENOMEM; + + iommu->maxpages = size / IOMMU_PAGE_SIZE; + pr_info("iommu membase %p pgtbase %p pgtsize %llx maxpages %lx\n", iommu->membase, iommu->pgtbase, size, iommu->maxpages); + tmp = MAX_DOMAIN_ID / 8; + bitmap_sz = (MAX_DOMAIN_ID % 8) ? (tmp + 1) : tmp; + iommu->domain_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL); + if (iommu->domain_bitmap == NULL) { + pr_err("Loongson-IOMMU: domain bitmap alloc err bitmap_sz:%d\n", bitmap_sz); + goto out_err; + } + + tmp = MAX_ATTACHED_DEV_ID / 8; + bitmap_sz = (MAX_ATTACHED_DEV_ID % 8) ? (tmp + 1) : tmp; + iommu->devtable_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL); + if (iommu->devtable_bitmap == NULL) { + pr_err("Loongson-IOMMU: devtable bitmap alloc err bitmap_sz:%d\n", bitmap_sz); + goto out_err_1; + } + + tmp = iommu->maxpages / 8; + bitmap_sz = (iommu->maxpages % 8) ? (tmp + 1) : tmp; + iommu->pgtable_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL); + if (iommu->pgtable_bitmap == NULL) { + pr_err("Loongson-IOMMU: pgtable bitmap alloc err bitmap_sz:%d\n", bitmap_sz); + goto out_err_2; + } + + bus_set_iommu(&pci_bus_type, &loongson_iommu_ops); + + return 0; + +out_err_2: + kfree(iommu->devtable_bitmap); + iommu->devtable_bitmap = NULL; +out_err_1: + kfree(iommu->domain_bitmap); + iommu->domain_bitmap = NULL; +out_err: + + return ret; +} + +static void loongson_iommu_remove(struct pci_dev *pdev) +{ + struct loongson_iommu *iommu = NULL; + + iommu = loongarch_get_iommu(pdev); + if (iommu == NULL) + return; + + if (iommu->domain_bitmap != NULL) { + kfree(iommu->domain_bitmap); + iommu->domain_bitmap = NULL; + } + + if (iommu->devtable_bitmap != NULL) { + kfree(iommu->devtable_bitmap); + iommu->devtable_bitmap = NULL; + } + + if (iommu->pgtable_bitmap != NULL) { + kfree(iommu->pgtable_bitmap); + iommu->pgtable_bitmap = NULL; + } + + iommu->membase = NULL; + iommu->pgtbase = NULL; +} + +static int __init check_ivrs_checksum(struct acpi_table_header *table) +{ + int i; + u8 checksum = 0, *p = (u8 *)table; + + for (i = 0; i < table->length; ++i) + checksum += p[i]; + if (checksum != 0) { + /* ACPI table corrupt */ + pr_err("IVRS invalid checksum\n"); + return -ENODEV; + } + + return 0; +} + +struct loongson_iommu_rlookup_entry *create_rlookup_entry(int pcisegment) +{ + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + rlookupentry = kzalloc(sizeof(struct loongson_iommu_rlookup_entry), GFP_KERNEL); + if (rlookupentry == NULL) + return rlookupentry; + + rlookupentry->pcisegment = pcisegment; + + /* IOMMU rlookup table - find the IOMMU for a specific device */ + rlookupentry->loongson_iommu_rlookup_table = (void *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, get_order(rlookup_table_size)); + if (rlookupentry->loongson_iommu_rlookup_table == NULL) { + kfree(rlookupentry); + rlookupentry = NULL; + } else { + list_add(&rlookupentry->list, &loongson_rlookup_iommu_list); + } + + return rlookupentry; +} + +/* Writes the specific IOMMU for a device into the rlookup table */ +static void __init set_iommu_for_device(loongson_iommu *iommu, + u16 devid) +{ + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + rlookupentry = lookup_rlooptable(iommu->segment); + if (rlookupentry == NULL) + rlookupentry = create_rlookup_entry(iommu->segment); + + if (rlookupentry != NULL) + rlookupentry->loongson_iommu_rlookup_table[devid] = iommu; +} + +static inline u32 get_ivhd_header_size(struct ivhd_header *h) +{ + u32 size = 0; + + switch (h->type) { + case IVHD_HEAD_TYPE10: + size = 24; + break; + case IVHD_HEAD_TYPE11: + case IVHD_HEAD_TYPE40: + size = 40; + break; + } + return size; +} + +static inline void update_last_devid(u16 devid) +{ + if (devid > loongson_iommu_last_bdf) + loongson_iommu_last_bdf = devid; +} + +/* + * This function calculates the length of a given IVHD entry + */ +static inline int ivhd_entry_length(u8 *ivhd) +{ + u32 type = ((struct ivhd_entry *)ivhd)->type; + + if (type < 0x80) { + return 0x04 << (*ivhd >> 6); + } else if (type == IVHD_DEV_ACPI_HID) { + /* For ACPI_HID, offset 21 is uid len */ + return *((u8 *)ivhd + 21) + 22; + } + return 0; +} + +/* + * After reading the highest device id from the IOMMU PCI capability header + * this function looks if there is a higher device id defined in the ACPI table + */ +static int __init find_last_devid_from_ivhd(struct ivhd_header *h) +{ + u8 *p = (void *)h, *end = (void *)h; + struct ivhd_entry *dev; + + u32 ivhd_size = get_ivhd_header_size(h); + + if (!ivhd_size) { + pr_err("loongson-iommu: Unsupported IVHD type %#x\n", h->type); + return -EINVAL; + } + + p += ivhd_size; + end += h->length; + + while (p < end) { + dev = (struct ivhd_entry *)p; + switch (dev->type) { + case IVHD_DEV_ALL: + /* Use maximum BDF value for DEV_ALL */ + update_last_devid(MAX_BDF_NUM); + break; + case IVHD_DEV_SELECT: + case IVHD_DEV_RANGE_END: + case IVHD_DEV_ALIAS: + case IVHD_DEV_EXT_SELECT: + /* all the above subfield types refer to device ids */ + update_last_devid(dev->devid); + break; + default: + break; + } + p += ivhd_entry_length(p); + } + + WARN_ON(p != end); + + return 0; +} + +/* + * Iterate over all IVHD entries in the ACPI table and find the highest device + * id which we need to handle. This is the first of three functions which parse + * the ACPI table. So we check the checksum here. + */ +static int __init find_last_devid_acpi(struct acpi_table_header *table) +{ + u8 *p = (u8 *)table, *end = (u8 *)table; + struct ivhd_header *h; + + p += IVRS_HEADER_LENGTH; + + end += table->length; + while (p < end) { + h = (struct ivhd_header *)p; + if (h->type == loongson_iommu_target_ivhd_type) { + int ret = find_last_devid_from_ivhd(h); + + if (ret) + return ret; + } + + if (h->length == 0) + break; + + p += h->length; + } + + if (p != end) + return -EINVAL; + + + return 0; +} + +/* + * Takes a pointer to an loongarch IOMMU entry in the ACPI table and + * initializes the hardware and our data structures with it. + */ +static int __init init_iommu_from_acpi(loongson_iommu *iommu, + struct ivhd_header *h) +{ + u8 *p = (u8 *)h; + u8 *end = p; + u16 devid = 0, devid_start = 0; + u32 dev_i, ivhd_size; + struct ivhd_entry *e; + + /* + * Done. Now parse the device entries + */ + ivhd_size = get_ivhd_header_size(h); + if (!ivhd_size) { + pr_err("loongarch iommu: Unsupported IVHD type %#x\n", h->type); + return -EINVAL; + } + + if (h->length == 0) + return -EINVAL; + + p += ivhd_size; + end += h->length; + + while (p < end) { + e = (struct ivhd_entry *)p; + switch (e->type) { + case IVHD_DEV_ALL: + for (dev_i = 0; dev_i <= loongson_iommu_last_bdf; ++dev_i) + set_iommu_for_device(iommu, dev_i); + break; + + case IVHD_DEV_SELECT: + pr_info(" DEV_SELECT\t\t\t devid: %02x:%02x.%x\n", + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid)); + + devid = e->devid; + set_iommu_for_device(iommu, devid); + break; + + case IVHD_DEV_SELECT_RANGE_START: + pr_info(" DEV_SELECT_RANGE_START\t devid: %02x:%02x.%x\n", + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid)); + + devid_start = e->devid; + break; + + case IVHD_DEV_RANGE_END: + pr_info(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid)); + + devid = e->devid; + for (dev_i = devid_start; dev_i <= devid; ++dev_i) + set_iommu_for_device(iommu, dev_i); + break; + + default: + break; + } + + p += ivhd_entry_length(p); + } + + return 0; +} + +/* + * This function clues the initialization function for one IOMMU + * together and also allocates the command buffer and programs the + * hardware. It does NOT enable the IOMMU. This is done afterwards. + */ +static int __init init_iommu_one(loongson_iommu *iommu, struct ivhd_header *h) +{ + int ret; + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + spin_lock_init(&iommu->domain_bitmap_lock); + spin_lock_init(&iommu->dom_info_lock); + spin_lock_init(&iommu->pgtable_bitmap_lock); + mutex_init(&iommu->loongson_iommu_pgtlock); + + /* Add IOMMU to internal data structures */ + INIT_LIST_HEAD(&iommu->dom_list); + + list_add_tail(&iommu->list, &loongson_iommu_list); + + /* + * Copy data from ACPI table entry to the iommu struct + */ + iommu->devid = h->devid; + iommu->segment = h->pci_seg; + + ret = init_iommu_from_acpi(iommu, h); + if (ret) { + pr_err("%s init iommu from acpi failed\n", __func__); + return ret; + } + + rlookupentry = lookup_rlooptable(iommu->segment); + if (rlookupentry != NULL) { + /* + * Make sure IOMMU is not considered to translate itself. + * The IVRS table tells us so, but this is a lie! + */ + rlookupentry->loongson_iommu_rlookup_table[iommu->devid] = NULL; + } + + return 0; +} + +/* + * Iterates over all IOMMU entries in the ACPI table, allocates the + * IOMMU structure and initializes it with init_iommu_one() + */ +static int __init init_iommu_all(struct acpi_table_header *table) +{ + int ret; + u8 *p = (u8 *)table, *end = (u8 *)table; + struct ivhd_header *h; + loongson_iommu *iommu; + + end += table->length; + p += IVRS_HEADER_LENGTH; + + while (p < end) { + h = (struct ivhd_header *)p; + + if (h->length == 0) + break; + + if (*p == loongson_iommu_target_ivhd_type) { + pr_info("device: %02x:%02x.%01x seg: %d\n", + PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), PCI_FUNC(h->devid), h->pci_seg); + + iommu = kzalloc(sizeof(loongson_iommu), GFP_KERNEL); + if (iommu == NULL) { + pr_info("%s alloc memory for iommu failed\n", __func__); + return -ENOMEM; + } + + ret = init_iommu_one(iommu, h); + if (ret) { + kfree(iommu); + pr_info("%s init iommu failed\n", __func__); + return ret; + } + } + p += h->length; + } + + if (p != end) + return -EINVAL; + + return 0; +} + +/** + * get_highest_supported_ivhd_type - Look up the appropriate IVHD type + * @ivrs Pointer to the IVRS header + * + * This function search through all IVDB of the maximum supported IVHD + */ +static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) +{ + u8 *base = (u8 *)ivrs; + struct ivhd_header *ivhd = (struct ivhd_header *)(base + IVRS_HEADER_LENGTH); + u8 last_type = ivhd->type; + u16 devid = ivhd->devid; + + while (((u8 *)ivhd - base < ivrs->length) && + (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED) && + (ivhd->length > 0)) { + u8 *p = (u8 *) ivhd; + + if (ivhd->devid == devid) + last_type = ivhd->type; + ivhd = (struct ivhd_header *)(p + ivhd->length); + } + + return last_type; +} + +static inline unsigned long tbl_size(int entry_size) +{ + unsigned int shift = PAGE_SHIFT + + get_order(((int)loongson_iommu_last_bdf + 1) * entry_size); + + return 1UL << shift; +} + +static int __init loongson_iommu_ivrs_init(void) +{ + int ret = 0; + acpi_status status; + struct acpi_table_header *ivrs_base; + + status = acpi_get_table("IVRS", 0, &ivrs_base); + if (status == AE_NOT_FOUND) { + pr_info("%s get ivrs table failed\n", __func__); + return -ENODEV; + } + + /* + * Validate checksum here so we don't need to do it when + * we actually parse the table + */ + ret = check_ivrs_checksum(ivrs_base); + if (ret) + goto out; + + loongson_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); + pr_info("Using IVHD type %#x\n", loongson_iommu_target_ivhd_type); + + /* + * First parse ACPI tables to find the largest Bus/Dev/Func + * we need to handle. Upon this information the shared data + * structures for the IOMMUs in the system will be allocated + */ + ret = find_last_devid_acpi(ivrs_base); + if (ret) { + pr_err("%s find last devid failed\n", __func__); + goto out; + } + + rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); + + /* + * now the data structures are allocated and basically initialized + * start the real acpi table scan + */ + ret = init_iommu_all(ivrs_base); + +out: + /* Don't leak any ACPI memory */ + acpi_put_table(ivrs_base); + ivrs_base = NULL; + + return ret; +} + +static int __init loongson_iommu_ivrs_init_stub(void) +{ + u32 dev_i; + loongson_iommu *iommu; + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + /* Use maximum BDF value for DEV_ALL */ + update_last_devid(MAX_BDF_NUM); + + rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); + + iommu = kzalloc(sizeof(loongson_iommu), GFP_KERNEL); + if (iommu == NULL) { + pr_info("%s alloc memory for iommu failed\n", __func__); + return -ENOMEM; + } + + spin_lock_init(&iommu->domain_bitmap_lock); + spin_lock_init(&iommu->dom_info_lock); + spin_lock_init(&iommu->pgtable_bitmap_lock); + mutex_init(&iommu->loongson_iommu_pgtlock); + + /* Add IOMMU to internal data structures */ + INIT_LIST_HEAD(&iommu->dom_list); + + list_add_tail(&iommu->list, &loongson_iommu_list); + + /* + * Copy data from ACPI table entry to the iommu struct + */ + iommu->devid = 0xd0; + iommu->segment = 0; + + for (dev_i = 0; dev_i <= loongson_iommu_last_bdf; ++dev_i) + set_iommu_for_device(iommu, dev_i); + + rlookupentry = lookup_rlooptable(iommu->segment); + if (rlookupentry != NULL) { + /* + * Make sure IOMMU is not considered to translate itself. + * The IVRS table tells us so, but this is a lie! + */ + rlookupentry->loongson_iommu_rlookup_table[iommu->devid] = NULL; + } + + return 0; +} + +static void free_iommu_rlookup_entry(void) +{ + loongson_iommu *iommu = NULL; + struct loongson_iommu_rlookup_entry *rlookupentry = NULL; + + while (!list_empty(&loongson_iommu_list)) { + iommu = list_first_entry(&loongson_iommu_list, loongson_iommu, list); + list_del(&iommu->list); + kfree(iommu); + } + + while (!list_empty(&loongson_rlookup_iommu_list)) { + rlookupentry = list_first_entry(&loongson_rlookup_iommu_list, + struct loongson_iommu_rlookup_entry, list); + + list_del(&rlookupentry->list); + if (rlookupentry->loongson_iommu_rlookup_table != NULL) { + free_pages( + (unsigned long)rlookupentry->loongson_iommu_rlookup_table, + get_order(rlookup_table_size)); + + rlookupentry->loongson_iommu_rlookup_table = NULL; + } + + kfree(rlookupentry); + } +} + +static int __init loonson_iommu_setup(char *str) +{ + if (!str) + return -EINVAL; + + while (*str) { + if (!strncmp(str, "on", 2)) { + loongson_iommu_disable = 0; + pr_info("IOMMU enabled\n"); + } else if (!strncmp(str, "off", 3)) { + loongson_iommu_disable = 1; + pr_info("IOMMU disabled\n"); + } + str += strcspn(str, ","); + while (*str == ',') + str++; + } + return 0; +} +__setup("loongson_iommu=", loonson_iommu_setup); + +static const struct pci_device_id loongson_iommu_pci_tbl[] = { + { PCI_DEVICE(0x14, 0x7a1f) }, + { 0, } +}; + +static struct pci_driver loongson_iommu_driver = { + .name = "loongson-iommu", + .probe = loongson_iommu_probe, + .remove = loongson_iommu_remove, + .id_table = loongson_iommu_pci_tbl, +}; + +static int __init loongson_iommu_driver_init(void) +{ + int ret = 0; + + if (!loongson_iommu_disable) { + ret = loongson_iommu_ivrs_init(); + if (ret < 0) { + free_iommu_rlookup_entry(); + pr_err("Failed to init iommu by ivrs\n"); + ret = loongson_iommu_ivrs_init_stub(); + if (ret < 0) { + free_iommu_rlookup_entry(); + pr_err("Failed to init iommu by stub\n"); + return ret; + } + } + + ret = pci_register_driver(&loongson_iommu_driver); + if (ret < 0) { + pr_err("Failed to register IOMMU driver\n"); + return ret; + } + } + + return ret; +} + +static void __exit loongson_iommu_driver_exit(void) +{ + if (!loongson_iommu_disable) { + free_iommu_rlookup_entry(); + pci_unregister_driver(&loongson_iommu_driver); + } +} + +module_init(loongson_iommu_driver_init); +module_exit(loongson_iommu_driver_exit); diff --git a/drivers/iommu/loongson/iommu.h b/drivers/iommu/loongson/iommu.h new file mode 100644 index 0000000000000000000000000000000000000000..d50eb541f28d00b3a310906e450205f273eab972 --- /dev/null +++ b/drivers/iommu/loongson/iommu.h @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Loongson IOMMU Driver + * + * Copyright (C) 2020-2021 Loongson Technology Ltd. + * Author: Lv Chen + * Wang Yang + */ + +#ifndef LOONGSON_IOMMU_H +#define LOONGSON_IOMMU_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#define IOVA_WIDTH 47 + +/* Bit value definition for I/O PTE fields */ +#define IOMMU_PTE_PR (1ULL << 0) /* Present */ +#define IOMMU_PTE_HP (1ULL << 1) /* HugePage */ +#define IOMMU_PTE_IR (1ULL << 2) /* Readable */ +#define IOMMU_PTE_IW (1ULL << 3) /* Writeable */ +#define IOMMU_PTE_RW (IOMMU_PTE_PR | IOMMU_PTE_IR | IOMMU_PTE_IW) + +#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR) +#define IOMMU_PTE_HUGEPAGE(pte) ((pte) & IOMMU_PTE_HP) + +#define iommu_pt_present(shadow_entry) ((*shadow_entry != 0)) + +/* + * shadow_entry using kmalloc to request memory space, + * the memory address requested by kmalloc is ARCH_DMA_MINALIGN-aligned, + * when the shadow_entry address is not a ARCH_DMA_MINALIGN-aligned + * address, we think that shadow_entry store large pages + */ +#define iommu_pt_huge(shd_entry_ptr) ((*shd_entry_ptr) & IOMMU_PTE_HP) + +#define LA_IOMMU_PGSIZE (SZ_16K | SZ_32M) + +#define IOMMU_PT_LEVEL0 0x00 +#define IOMMU_PT_LEVEL1 0x01 + +/* IOMMU page table */ +#define IOMMU_PAGE_SHIFT PAGE_SHIFT +#define IOMMU_PAGE_SIZE (_AC(1, UL) << IOMMU_PAGE_SHIFT) +#define IOMMU_LEVEL_STRIDE (IOMMU_PAGE_SHIFT - 3) +#define IOMMU_PTRS_PER_LEVEL (IOMMU_PAGE_SIZE >> 3) +#define IOMMU_LEVEL_SHIFT(n) (((n) * IOMMU_LEVEL_STRIDE) + IOMMU_PAGE_SHIFT) +#define IOMMU_LEVEL_SIZE(n) (_AC(1, UL) << (((n) * IOMMU_LEVEL_STRIDE) + IOMMU_PAGE_SHIFT)) +#define IOMMU_LEVEL_MASK(n) (~(IOMMU_LEVEL_SIZE(n) - 1)) +#define IOMMU_LEVEL_MAX DIV_ROUND_UP((IOVA_WIDTH - IOMMU_PAGE_SHIFT), IOMMU_LEVEL_STRIDE) +#define IOMMU_PAGE_MASK (~(IOMMU_PAGE_SIZE - 1)) + +#define IOMMU_HPAGE_SIZE (1UL << IOMMU_LEVEL_SHIFT(IOMMU_PT_LEVEL1)) +#define IOMMU_HPAGE_MASK (~(IOMMU_HPAGE_SIZE - 1)) + +/* Virtio page use size of 16k */ +#define LA_VIRTIO_PAGE_SHIFT 14 +#define LA_VIRTIO_PAGE_SIZE (_AC(1, UL) << LA_VIRTIO_PAGE_SHIFT) +#define LA_VIRTIO_PAGE_MASK (~((1ULL << LA_VIRTIO_PAGE_SHIFT) - 1)) + +/* Bits of iommu map address space field */ +#define LA_IOMMU_PFN_LO 0x0 +#define PFN_LO_SHIFT 12 +#define LA_IOMMU_PFN_HI 0x4 +#define PFN_HI_MASK 0x3ffff +#define LA_IOMMU_VFN_LO 0x8 +#define VFN_LO_SHIFT 12 +#define LA_IOMMU_VFN_HI 0xC +#define VFN_HI_MASK 0x3ffff + +/* wired | index | domain | shift */ +#define LA_IOMMU_WIDS 0x10 +/* valid | busy | tlbar/aw | cmd */ +#define LA_IOMMU_VBTC 0x14 +#define IOMMU_PGTABLE_BUSY (1 << 16) +/* enable |index | valid | domain | bdf */ +#define LA_IOMMU_EIVDB 0x18 +/* enable | valid | cmd */ +#define LA_IOMMU_CMD 0x1C +#define LA_IOMMU_PGD0_LO 0x20 +#define LA_IOMMU_PGD0_HI 0x24 +#define STEP_PGD 0x8 +#define STEP_PGD_SHIFT 3 +#define LA_IOMMU_PGD_LO(domain_id) \ + (LA_IOMMU_PGD0_LO + ((domain_id) << STEP_PGD_SHIFT)) +#define LA_IOMMU_PGD_HI(domain_id) \ + (LA_IOMMU_PGD0_HI + ((domain_id) << STEP_PGD_SHIFT)) + +#define LA_IOMMU_DIR_CTRL0 0xA0 +#define LA_IOMMU_DIR_CTRL1 0xA4 +#define LA_IOMMU_DIR_CTRL(x) (LA_IOMMU_DIR_CTRL0 + ((x) << 2)) + +#define LA_IOMMU_SAFE_BASE_HI 0xE0 +#define LA_IOMMU_SAFE_BASE_LO 0xE4 +#define LA_IOMMU_EX_ADDR_LO 0xE8 +#define LA_IOMMU_EX_ADDR_HI 0xEC + +#define LA_IOMMU_PFM_CNT_EN 0x100 + +#define LA_IOMMU_RD_HIT_CNT_0 0x110 +#define LA_IOMMU_RD_MISS_CNT_O 0x114 +#define LA_IOMMU_WR_HIT_CNT_0 0x118 +#define LA_IOMMU_WR_MISS_CNT_0 0x11C +#define LA_IOMMU_RD_HIT_CNT_1 0x120 +#define LA_IOMMU_RD_MISS_CNT_1 0x124 +#define LA_IOMMU_WR_HIT_CNT_1 0x128 +#define LA_IOMMU_WR_MISS_CNT_1 0x12C +#define LA_IOMMU_RD_HIT_CNT_2 0x130 +#define LA_IOMMU_RD_MISS_CNT_2 0x134 +#define LA_IOMMU_WR_HIT_CNT_2 0x138 +#define LA_IOMMU_WR_MISS_CNT_2 0x13C + +#define MAX_DOMAIN_ID 16 +#define MAX_ATTACHED_DEV_ID 16 +#define MAX_PAGES_NUM (SZ_128M / IOMMU_PAGE_SIZE) + +#define iommu_ptable_end(addr, end, level) \ +({ unsigned long __boundary = ((addr) + IOMMU_LEVEL_SIZE(level)) & IOMMU_LEVEL_MASK(level); \ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +/* To find an entry in an iommu page table directory */ +#define iommu_shadow_index(addr, level) \ + (((addr) >> ((level * IOMMU_LEVEL_STRIDE) + IOMMU_PAGE_SHIFT)) & (IOMMU_PTRS_PER_LEVEL - 1)) + +/* IOMMU iommu_table entry */ +typedef struct { unsigned long iommu_pte; } iommu_pte; + +typedef struct loongson_iommu { + struct list_head list; + spinlock_t domain_bitmap_lock; /* Lock for domain allocing */ + spinlock_t dom_info_lock; /* Lock for priv->list */ + spinlock_t pgtable_bitmap_lock; /* Lock for bitmap of page table */ + struct mutex loongson_iommu_pgtlock; /* Lock for iommu page table */ + void *domain_bitmap; /* Bitmap of global domains */ + void *devtable_bitmap; /* Bitmap of devtable */ + void *pgtable_bitmap; /* Bitmap of devtable pages for page table */ + struct list_head dom_list; /* List of all domain privates */ + u16 devid; /* PCI device id of the IOMMU device */ + int segment; /* PCI segment# */ + void *membase; + void *pgtbase; + unsigned long maxpages; +} loongson_iommu; + +struct loongson_iommu_rlookup_entry +{ + struct list_head list; + struct loongson_iommu **loongson_iommu_rlookup_table; + int pcisegment; +}; + +/* shadow page table entry */ +typedef struct spt_entry { + unsigned long *gmem_ptable; /* gmemory entry address base*/ + unsigned long *shadow_ptable; /* virtual address base for shadow page */ + int index; /* index 128M gmem */ + int dirty; + int present; +} spt_entry; + +typedef struct iommu_info { + struct list_head list; /* for loongson_iommu_pri->iommu_devlist */ + spt_entry *shadow_pgd; + struct loongson_iommu *iommu; + spinlock_t devlock; /* priv dev list lock */ + struct list_head dev_list; /* List of all devices in this domain iommu */ + unsigned int dev_cnt; /* devices assigned to this domain iommu */ + short id; +} iommu_info; + +/* One vm is equal to a domain, one domain has a priv */ +typedef struct dom_info { + struct list_head list; /* For list of all domains */ + struct list_head iommu_devlist; + struct iommu_domain domain; + void *mmio_pgd; /* 0x10000000~0x8fffffff */ + spinlock_t lock; /* Lock for priv->iommu_devlist */ +} dom_info; + +/* A device for passthrough */ +struct loongson_iommu_dev_data { + struct list_head list; /* for iommu_entry->dev_list */ + struct loongson_iommu *iommu; + iommu_info *iommu_entry; + unsigned short bdf; + int count; + int index; /* index in device table */ +}; + +static inline unsigned long iommu_pgt_v2p(loongson_iommu *iommu, void *va) +{ + return (unsigned long)(va - iommu->pgtbase); +} + +static inline unsigned long *iommu_ptable_offset(unsigned long *table_entry, + unsigned long addr, int level) +{ + return table_entry + iommu_shadow_index(addr, level); +} + +static inline unsigned long *iommu_shadow_offset(spt_entry *shadow_entry, + unsigned long addr, int level) +{ + unsigned long *table_base; + + table_base = shadow_entry->shadow_ptable; + + return table_base + iommu_shadow_index(addr, level); +} + +#endif /* LOONGSON_IOMMU_H */ diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 3c24bf45263ce80b99bbc3193296e8b22b5c9d6a..641d206326abb1700e580d53569e0011d63e9f93 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -545,6 +545,12 @@ config EXYNOS_IRQ_COMBINER Say yes here to add support for the IRQ combiner devices embedded in Samsung Exynos chips. +config IRQ_LOONGARCH_CPU + bool + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN + select GENERIC_IRQ_EFFECTIVE_AFF_MASK + config LOONGSON_LIOINTC bool "Loongson Local I/O Interrupt Controller" depends on MACH_LOONGSON64 @@ -554,9 +560,19 @@ config LOONGSON_LIOINTC help Support for the Loongson Local I/O Interrupt Controller. +config LOONGSON_EIOINTC + bool "Loongson Extend I/O Interrupt Controller" + depends on LOONGARCH + depends on MACH_LOONGSON64 + default MACH_LOONGSON64 + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_CHIP + help + Support for the Loongson3 Extend I/O Interrupt Vector Controller. + config LOONGSON_HTPIC bool "Loongson3 HyperTransport PIC Controller" - depends on MACH_LOONGSON64 + depends on (MACH_LOONGSON64 && MIPS) default y select IRQ_DOMAIN select GENERIC_IRQ_CHIP @@ -564,12 +580,12 @@ config LOONGSON_HTPIC Support for the Loongson-3 HyperTransport PIC Controller. config LOONGSON_HTVEC - bool "Loongson3 HyperTransport Interrupt Vector Controller" + bool "Loongson HyperTransport Interrupt Vector Controller" depends on MACH_LOONGSON64 default MACH_LOONGSON64 select IRQ_DOMAIN_HIERARCHY help - Support for the Loongson3 HyperTransport Interrupt Vector Controller. + Support for the Loongson HyperTransport Interrupt Vector Controller. config LOONGSON_PCH_PIC bool "Loongson PCH PIC Controller" @@ -590,6 +606,14 @@ config LOONGSON_PCH_MSI help Support for the Loongson PCH MSI Controller. +config LOONGSON_PCH_LPC + bool "Loongson PCH LPC Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + default (MACH_LOONGSON64 && LOONGARCH) + select IRQ_DOMAIN_HIERARCHY + help + Support for the Loongson PCH LPC Controller. + config MST_IRQ bool "MStar Interrupt Controller" depends on ARCH_MEDIATEK || ARCH_MSTARV7 || COMPILE_TEST diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 94c2885882ee195cf37099aa6cab8105a7414949..44794b04e82a70cffa0050e9a9ecc2cf8f54971e 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -107,10 +107,13 @@ obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o obj-$(CONFIG_TI_PRUSS_INTC) += irq-pruss-intc.o +obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o +obj-$(CONFIG_LOONGSON_EIOINTC) += irq-loongson-eiointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o obj-$(CONFIG_LOONGSON_HTVEC) += irq-loongson-htvec.o obj-$(CONFIG_LOONGSON_PCH_PIC) += irq-loongson-pch-pic.o obj-$(CONFIG_LOONGSON_PCH_MSI) += irq-loongson-pch-msi.o +obj-$(CONFIG_LOONGSON_PCH_LPC) += irq-loongson-pch-lpc.o obj-$(CONFIG_MST_IRQ) += irq-mst-intc.o obj-$(CONFIG_SL28CPLD_INTC) += irq-sl28cpld.o diff --git a/drivers/irqchip/irq-loongarch-cpu.c b/drivers/irqchip/irq-loongarch-cpu.c new file mode 100644 index 0000000000000000000000000000000000000000..743015d0e26f83078880affcc9e9284ca8259aac --- /dev/null +++ b/drivers/irqchip/irq-loongarch-cpu.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technologies, Inc. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +static struct irq_domain *irq_domain; + +static void mask_loongarch_irq(struct irq_data *d) +{ + clear_csr_ecfg(ECFGF(d->hwirq)); +} + +static void unmask_loongarch_irq(struct irq_data *d) +{ + set_csr_ecfg(ECFGF(d->hwirq)); +} + +static struct irq_chip cpu_irq_controller = { + .name = "LoongArch", + .irq_mask = mask_loongarch_irq, + .irq_unmask = unmask_loongarch_irq, +}; + +static void handle_cpu_irq(struct pt_regs *regs) +{ + int hwirq; + unsigned int estat = read_csr_estat() & CSR_ESTAT_IS; + + while ((hwirq = ffs(estat))) { + estat &= ~BIT(hwirq - 1); + handle_domain_irq(irq_domain, hwirq - 1, regs); + } +} + +int get_ipi_irq(void) +{ + return irq_create_mapping(irq_domain, INT_IPI); +} + +int get_pmc_irq(void) +{ + return irq_create_mapping(irq_domain, INT_PCOV); +} + +int get_timer_irq(void) +{ + return irq_create_mapping(irq_domain, INT_TI); +} + +static int loongarch_cpu_intc_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_noprobe(irq); + irq_set_chip_and_handler(irq, &cpu_irq_controller, handle_percpu_irq); + + return 0; +} + +static const struct irq_domain_ops loongarch_cpu_intc_irq_domain_ops = { + .map = loongarch_cpu_intc_map, + .xlate = irq_domain_xlate_onecell, +}; + +struct irq_domain * __init loongarch_cpu_irq_init(void) +{ + struct fwnode_handle *domain_handle; + + /* Mask interrupts. */ + clear_csr_ecfg(ECFG0_IM); + clear_csr_estat(ESTATF_IP); + + domain_handle = irq_domain_alloc_fwnode(NULL); + irq_domain = irq_domain_create_linear(domain_handle, EXCCODE_INT_NUM, + &loongarch_cpu_intc_irq_domain_ops, NULL); + + if (!irq_domain) + panic("Failed to add irqdomain for LoongArch CPU"); + + set_handle_irq(&handle_cpu_irq); + + return irq_domain; +} diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c new file mode 100644 index 0000000000000000000000000000000000000000..11c402e2b8944f12a2e3e982586ea9348e4974ee --- /dev/null +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Jianmin Lv + * Loongson Extend I/O Interrupt Vector support + */ + +#define pr_fmt(fmt) "eiointc: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EIOINTC_REG_NODEMAP 0x14a0 +#define EIOINTC_REG_IPMAP 0x14c0 +#define EIOINTC_REG_ENABLE 0x1600 +#define EIOINTC_REG_BOUNCE 0x1680 +#define EIOINTC_REG_ISR 0x1800 +#define EIOINTC_REG_ROUTE 0x1c00 + +#define VEC_REG_COUNT 4 +#define VEC_COUNT_PER_REG 64 +#define VEC_COUNT (VEC_REG_COUNT * VEC_COUNT_PER_REG) +#define VEC_REG_IDX(irq_id) ((irq_id) / VEC_COUNT_PER_REG) +#define VEC_REG_BIT(irq_id) ((irq_id) % VEC_COUNT_PER_REG) +#define EIOINTC_ALL_ENABLE 0xffffffff + +#define MAX_EIO_NODES (NR_CPUS / CORES_PER_EIO_NODE) + +static int nr_pics; + +struct eiointc_priv { + u32 node; + nodemask_t node_map; + cpumask_t cpuspan_map; + struct fwnode_handle *domain_handle; + struct irq_domain *eiointc_domain; +}; + +static struct eiointc_priv *eiointc_priv[MAX_IO_PICS]; + +int eiointc_get_node(int id) +{ + return eiointc_priv[id]->node; +} + +static int cpu_to_eio_node(int cpu) +{ + return cpu_logical_map(cpu) / CORES_PER_EIO_NODE; +} + +static unsigned int cpumask_nth(unsigned int idx, const struct cpumask *srcp) +{ + int cpu; + + for_each_cpu(cpu, srcp) + if (idx-- == 0) + return cpu; + + BUG(); +} + +static void eiointc_set_irq_route(int pos, unsigned int cpu, unsigned int mnode, nodemask_t *node_map) +{ + int i, node, cpu_node, route_node; + unsigned char coremap[MAX_EIO_NODES]; + uint32_t pos_off, data, data_byte, data_mask; + + pos_off = pos & ~3; + data_byte = pos & 3; + data_mask = ~BIT_MASK(data_byte) & 0xf; + + memset(coremap, 0, sizeof(unsigned char) * MAX_EIO_NODES); + + /* Calculate node and coremap of target irq */ + cpu_node = cpu_logical_map(cpu) / CORES_PER_EIO_NODE; + coremap[cpu_node] |= BIT(cpu_logical_map(cpu) % CORES_PER_EIO_NODE); + + for_each_online_cpu(i) { + node = cpu_to_eio_node(i); + if (!node_isset(node, *node_map)) + continue; + + /* EIO node 0 is in charge of inter-node interrupt dispatch */ + route_node = (node == mnode) ? cpu_node : node; + data = ((coremap[node] | (route_node << 4)) << (data_byte * 8)); + csr_any_send(EIOINTC_REG_ROUTE + pos_off, data, data_mask, node * CORES_PER_EIO_NODE); + } +} + +#ifdef CONFIG_LOONGARCH +static void virt_eiointc_set_irq_route(int pos, unsigned int cpu) +{ + iocsr_write8(cpu_logical_map(cpu), EIOINTC_REG_ROUTE + pos); +} +#endif + +static DEFINE_RAW_SPINLOCK(affinity_lock); + +static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *affinity, bool force) +{ + unsigned int cpu; + unsigned long flags; + uint32_t vector, regaddr; + struct cpumask online_affinity; + struct cpumask intersect_affinity; + struct eiointc_priv *priv = d->domain->host_data; + + if (!IS_ENABLED(CONFIG_SMP)) + return -EPERM; + + raw_spin_lock_irqsave(&affinity_lock, flags); + + cpumask_and(&online_affinity, affinity, cpu_online_mask); + if (cpumask_empty(&online_affinity)) { + raw_spin_unlock_irqrestore(&affinity_lock, flags); + return -EINVAL; + } + cpumask_and(&intersect_affinity, &online_affinity, &priv->cpuspan_map); + + if (!cpumask_empty(&intersect_affinity)) + cpu = cpumask_first(&intersect_affinity); + else { + int c, idx = 0; + struct cpumask complement_map; + struct cpumask cpuspan_online_map; + + cpu = cpumask_first(&online_affinity); + cpumask_complement(&complement_map, &priv->cpuspan_map); + cpumask_and(&cpuspan_online_map, &priv->cpuspan_map, cpu_online_mask); + + for_each_cpu(c, &complement_map) { + if (c == cpu) break; + idx++; + } + + idx = idx % cpumask_weight(&cpuspan_online_map); + cpu = cpumask_nth(idx, &cpuspan_online_map); + } + + if (!d->parent_data) + vector = d->hwirq; + else + vector = d->parent_data->hwirq; + + regaddr = EIOINTC_REG_ENABLE + ((vector >> 5) << 2); + + if (!cpu_has_hypervisor) { + /* Mask target vector */ + csr_any_send(regaddr, EIOINTC_ALL_ENABLE & (~BIT(vector & 0x1F)), + 0x0, priv->node * CORES_PER_EIO_NODE); + /* Set route for target vector */ + eiointc_set_irq_route(vector, cpu, priv->node, &priv->node_map); + /* Unmask target vector */ + csr_any_send(regaddr, EIOINTC_ALL_ENABLE, + 0x0, priv->node * CORES_PER_EIO_NODE); + } else { + iocsr_write32(EIOINTC_ALL_ENABLE & (~((1 << (vector & 0x1F)))), regaddr); + virt_eiointc_set_irq_route(vector, cpu); + iocsr_write32(EIOINTC_ALL_ENABLE, regaddr); + } + + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + raw_spin_unlock_irqrestore(&affinity_lock, flags); + + return IRQ_SET_MASK_OK; +} + +static int eiointc_index(int node) +{ + int i; + + for (i = 0; i < nr_pics; i++) { + if (node_isset(node, eiointc_priv[i]->node_map)) + return i; + } + + return -1; +} + +static int eiointc_router_init(unsigned int cpu) +{ + int i, bit; + int node = cpu_to_eio_node(cpu); + int index = eiointc_index(node); + uint32_t data; + + if (index < 0) { + pr_err("Error: invalid nodemap!\n"); + return -1; + } + + if ((cpu_logical_map(cpu) % CORES_PER_EIO_NODE) == 0) { + eiointc_enable(); + + for (i = 0; i < VEC_COUNT / 32; i++) { + data = (((1 << (i * 2 + 1)) << 16) | (1 << (i * 2))); + iocsr_write32(data, EIOINTC_REG_NODEMAP + i * 4); + } + + for (i = 0; i < VEC_COUNT / 32 / 4; i++) { + bit = BIT(1 + index); /* Route to IP[1 + index] */ + data = bit | (bit << 8) | (bit << 16) | (bit << 24); + iocsr_write32(data, EIOINTC_REG_IPMAP + i * 4); + } + + for (i = 0; i < VEC_COUNT / 4; i++) { + /* Route to Node-0 Core-0 */ + if (index == 0) + bit = BIT(cpu_logical_map(0)); + else + bit = (eiointc_priv[index]->node << 4) | 1; + if (cpu_has_hypervisor) + bit = cpu_logical_map(0); + + data = bit | (bit << 8) | (bit << 16) | (bit << 24); + iocsr_write32(data, EIOINTC_REG_ROUTE + i * 4); + } + + for (i = 0; i < VEC_COUNT / 32; i++) { + data = 0xffffffff; + iocsr_write32(data, EIOINTC_REG_ENABLE + i * 4); + iocsr_write32(data, EIOINTC_REG_BOUNCE + i * 4); + } + } + + return 0; +} + +static void eiointc_irq_dispatch(struct irq_desc *desc) +{ + int i; + u64 pending; + bool handled = false; + struct irq_chip *chip = irq_desc_get_chip(desc); + struct eiointc_priv *priv = irq_desc_get_handler_data(desc); + + chained_irq_enter(chip, desc); + + for (i = 0; i < VEC_REG_COUNT; i++) { + pending = iocsr_read64(EIOINTC_REG_ISR + (i << 3)); + iocsr_write64(pending, EIOINTC_REG_ISR + (i << 3)); + while (pending) { + int bit = __ffs(pending); + int virq = irq_linear_revmap(priv->eiointc_domain, bit + VEC_COUNT_PER_REG * i); + + generic_handle_irq(virq); + pending &= ~BIT(bit); + handled = true; + } + } + + if (!handled) + spurious_interrupt(); + + chained_irq_exit(chip, desc); +} + +static void eiointc_ack_irq(struct irq_data *d) +{ + if (d->parent_data) + irq_chip_ack_parent(d); +} + +static void eiointc_mask_irq(struct irq_data *d) +{ + if (d->parent_data) + irq_chip_mask_parent(d); +} + +static void eiointc_unmask_irq(struct irq_data *d) +{ + if (d->parent_data) + irq_chip_unmask_parent(d); +} + +static struct irq_chip eiointc_irq_chip = { + .name = "EIOINTC", + .irq_ack = eiointc_ack_irq, + .irq_mask = eiointc_mask_irq, + .irq_unmask = eiointc_unmask_irq, + .irq_set_affinity = eiointc_set_irq_affinity, +}; + +static int eiointc_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int ret; + unsigned int i, type; + unsigned long hwirq = 0; + struct eiointc *priv = domain->host_data; + + ret = irq_domain_translate_onecell(domain, arg, &hwirq, &type); + if (ret < 0) + return -EINVAL; + + if (hwirq >= IOCSR_EXTIOI_VECTOR_NUM) + return -EINVAL; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, &eiointc_irq_chip, + priv, handle_edge_irq, NULL, NULL); + } + + return 0; +} + +static void eiointc_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops eiointc_domain_ops = { + .translate = irq_domain_translate_onecell, + .alloc = eiointc_domain_alloc, + .free = eiointc_domain_free, +}; + +static int eiointc_suspend(void) +{ + return 0; +} + +static void eiointc_resume(void) +{ + int i, j; + struct irq_desc *desc; + struct irq_data *irq_data; + + eiointc_router_init(0); + + for (i = 0; i < nr_pics; i++) { + for (j = 0; j < VEC_COUNT; j++) { + desc = irq_to_desc(irq_find_mapping(eiointc_priv[i]->eiointc_domain, j)); + if (desc && desc->handle_irq && desc->handle_irq != handle_bad_irq) { + raw_spin_lock(&desc->lock); + irq_data = irq_domain_get_irq_data(eiointc_priv[i]->eiointc_domain, irq_desc_get_irq(desc)); + if (irq_data) eiointc_set_irq_affinity(irq_data, irq_data->common->affinity, 0); + raw_spin_unlock(&desc->lock); + } + } + } +} + +static struct syscore_ops eiointc_syscore_ops = { + .suspend = eiointc_suspend, + .resume = eiointc_resume, +}; + +struct irq_domain *eiointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_eio_pic *acpi_eiointc) +{ + int i, parent_irq; + unsigned long node_map; + struct eiointc_priv *priv; + + if (!acpi_eiointc) + return NULL; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; + + priv->domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_eiointc); + if (!priv->domain_handle) { + pr_err("Unable to allocate domain handle\n"); + goto out_free_priv; + } + + priv->node = acpi_eiointc->node; + node_map = acpi_eiointc->node_map ? : -1ULL; + + for_each_possible_cpu(i) { + if (node_map & (1ULL << cpu_to_eio_node(i))) { + node_set(cpu_to_eio_node(i), priv->node_map); + cpumask_or(&priv->cpuspan_map, &priv->cpuspan_map, cpumask_of(i)); + } + } + + /* Setup IRQ domain */ + priv->eiointc_domain = irq_domain_create_linear(priv->domain_handle, VEC_COUNT, + &eiointc_domain_ops, priv); + if (!priv->eiointc_domain) { + pr_err("loongson-eiointc: cannot add IRQ domain\n"); + goto out_free_priv; + } + + eiointc_priv[nr_pics++] = priv; + + eiointc_router_init(0); + + parent_irq = irq_create_mapping(parent, acpi_eiointc->cascade); + irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, priv); + + register_syscore_ops(&eiointc_syscore_ops); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING, + "irqchip/loongarch/intc:starting", + eiointc_router_init, NULL); + + return irq_find_matching_fwnode(priv->domain_handle, DOMAIN_BUS_ANY); + +out_free_priv: + priv->domain_handle = NULL; + kfree(priv); + + return NULL; +} diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index 6392aafb9a6318903912e167f2abda4d1f331c33..97c2584a9d23bd298c8c04f6fbd4e5eb17d469ee 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2020, Jiaxun Yang + * Jianmin Lv + * Huacai Chen * Loongson HyperTransport Interrupt Vector support */ @@ -16,22 +18,27 @@ #include #include #include +#include /* Registers */ #define HTVEC_EN_OFF 0x20 #define HTVEC_MAX_PARENT_IRQ 8 - #define VEC_COUNT_PER_REG 32 #define VEC_REG_IDX(irq_id) ((irq_id) / VEC_COUNT_PER_REG) #define VEC_REG_BIT(irq_id) ((irq_id) % VEC_COUNT_PER_REG) +#define HTVEC_SIZE 0x400 struct htvec { int num_parents; void __iomem *base; struct irq_domain *htvec_domain; raw_spinlock_t htvec_lock; + struct fwnode_handle *domain_handle; + u32 saved_vec_en[HTVEC_MAX_PARENT_IRQ]; }; +static struct htvec *htvec_priv; + static void htvec_irq_dispatch(struct irq_desc *desc) { int i; @@ -155,64 +162,137 @@ static void htvec_reset(struct htvec *priv) } } -static int htvec_of_init(struct device_node *node, - struct device_node *parent) +static int htvec_suspend(void) +{ + int i; + for (i = 0; i < htvec_priv->num_parents; i++) { + htvec_priv->saved_vec_en[i] = readl(htvec_priv->base + HTVEC_EN_OFF + 4 * i); + } + return 0; +} + +static void htvec_resume(void) +{ + int i; + for (i = 0; i < htvec_priv->num_parents; i++) { + writel(htvec_priv->saved_vec_en[i], htvec_priv->base + HTVEC_EN_OFF + 4 * i); + } +} + +static struct syscore_ops htvec_syscore_ops = { + .suspend = htvec_suspend, + .resume = htvec_resume, +}; + +static int htvec_init(phys_addr_t addr, unsigned long size, + int num_parents, int parent_irq[], struct fwnode_handle *domain_handle) { + int i; struct htvec *priv; - int err, parent_irq[8], i; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; + priv->num_parents = num_parents; + priv->base = ioremap(addr, size); + priv->domain_handle = domain_handle; raw_spin_lock_init(&priv->htvec_lock); - priv->base = of_iomap(node, 0); - if (!priv->base) { - err = -ENOMEM; - goto free_priv; - } - /* Interrupt may come from any of the 8 interrupt lines */ - for (i = 0; i < HTVEC_MAX_PARENT_IRQ; i++) { - parent_irq[i] = irq_of_parse_and_map(node, i); - if (parent_irq[i] <= 0) - break; - - priv->num_parents++; - } - - if (!priv->num_parents) { - pr_err("Failed to get parent irqs\n"); - err = -ENODEV; - goto iounmap_base; - } - - priv->htvec_domain = irq_domain_create_linear(of_node_to_fwnode(node), + /* Setup IRQ domain */ + priv->htvec_domain = irq_domain_create_linear(priv->domain_handle, (VEC_COUNT_PER_REG * priv->num_parents), &htvec_domain_ops, priv); if (!priv->htvec_domain) { - pr_err("Failed to create IRQ domain\n"); - err = -ENOMEM; - goto irq_dispose; + pr_err("loongson-htvec: cannot add IRQ domain\n"); + goto iounmap_base; } htvec_reset(priv); - for (i = 0; i < priv->num_parents; i++) + for (i = 0; i < priv->num_parents; i++) { irq_set_chained_handler_and_data(parent_irq[i], htvec_irq_dispatch, priv); + } + + htvec_priv = priv; + + register_syscore_ops(&htvec_syscore_ops); return 0; -irq_dispose: - for (; i > 0; i--) - irq_dispose_mapping(parent_irq[i - 1]); iounmap_base: iounmap(priv->base); -free_priv: + priv->domain_handle = NULL; kfree(priv); - return err; + return -EINVAL; +} + +#ifdef CONFIG_OF + +static int htvec_of_init(struct device_node *node, + struct device_node *parent) +{ + int i, err; + int parent_irq[8]; + int num_parents = 0; + struct resource res; + + if (of_address_to_resource(node, 0, &res)) + return -EINVAL; + + /* Interrupt may come from any of the 8 interrupt lines */ + for (i = 0; i < HTVEC_MAX_PARENT_IRQ; i++) { + parent_irq[i] = irq_of_parse_and_map(node, i); + if (parent_irq[i] <= 0) + break; + + num_parents++; + } + + err = htvec_init(res.start, resource_size(&res), + num_parents, parent_irq, of_node_to_fwnode(node)); + if (err < 0) + return err; + + return 0; } IRQCHIP_DECLARE(htvec, "loongson,htvec-1.0", htvec_of_init); + +#endif + +#ifdef CONFIG_ACPI + +struct irq_domain *htvec_acpi_init(struct irq_domain *parent, + struct acpi_madt_ht_pic *acpi_htvec) +{ + int i, ret; + int num_parents, parent_irq[8]; + struct fwnode_handle *domain_handle; + + if (!acpi_htvec) + return NULL; + + num_parents = HTVEC_MAX_PARENT_IRQ; + + domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_htvec); + if (!domain_handle) { + pr_err("Unable to allocate domain handle\n"); + return NULL; + } + + /* Interrupt may come from any of the 8 interrupt lines */ + for (i = 0; i < HTVEC_MAX_PARENT_IRQ; i++) + parent_irq[i] = irq_create_mapping(parent, acpi_htvec->cascade[i]); + + ret = htvec_init(acpi_htvec->address, acpi_htvec->size, + num_parents, parent_irq, domain_handle); + if (ret < 0) + return NULL; + + return irq_find_matching_fwnode(domain_handle, DOMAIN_BUS_ANY); +} + +#endif diff --git a/drivers/irqchip/irq-loongson-liointc.c b/drivers/irqchip/irq-loongson-liointc.c index 9ed1bc47366348b5b9066bc4873a317d82c011af..9643e18efcc39ea07f2da5e614c900e5b02db5d7 100644 --- a/drivers/irqchip/irq-loongson-liointc.c +++ b/drivers/irqchip/irq-loongson-liointc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2020, Jiaxun Yang + * Jianmin Lv * Loongson Local IO Interrupt Controller support */ @@ -16,13 +17,13 @@ #include #include -#include +#include #define LIOINTC_CHIP_IRQ 32 -#define LIOINTC_NUM_PARENT 4 +#define LIOINTC_NUM_PARENT 4 #define LIOINTC_INTC_CHIP_START 0x20 - +#define LIOINTC_MEM_SIZE 0x80 #define LIOINTC_REG_INTC_STATUS (LIOINTC_INTC_CHIP_START + 0x20) #define LIOINTC_REG_INTC_EN_STATUS (LIOINTC_INTC_CHIP_START + 0x04) #define LIOINTC_REG_INTC_ENABLE (LIOINTC_INTC_CHIP_START + 0x08) @@ -40,9 +41,12 @@ struct liointc_handler_data { }; struct liointc_priv { + struct fwnode_handle *domain_handle; struct irq_chip_generic *gc; struct liointc_handler_data handler[LIOINTC_NUM_PARENT]; u8 map_cache[LIOINTC_CHIP_IRQ]; + u32 int_pol; + u32 int_edge; bool has_lpc_irq_errata; }; @@ -51,11 +55,11 @@ static void liointc_chained_handle_irq(struct irq_desc *desc) struct liointc_handler_data *handler = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); struct irq_chip_generic *gc = handler->priv->gc; - u32 pending; + u32 pending, offset = cpu_logical_map(smp_processor_id()) * 8; chained_irq_enter(chip, desc); - pending = readl(gc->reg_base + LIOINTC_REG_INTC_STATUS); + pending = readl(gc->reg_base + LIOINTC_REG_INTC_STATUS + offset); if (!pending) { /* Always blame LPC IRQ if we have that bug */ @@ -123,6 +127,14 @@ static int liointc_set_type(struct irq_data *data, unsigned int type) return 0; } +static void liointc_suspend(struct irq_chip_generic *gc) +{ + struct liointc_priv *priv = gc->private; + + priv->int_pol = readl(gc->reg_base + LIOINTC_REG_INTC_POL); + priv->int_edge = readl(gc->reg_base + LIOINTC_REG_INTC_EDGE); +} + static void liointc_resume(struct irq_chip_generic *gc) { struct liointc_priv *priv = gc->private; @@ -135,72 +147,51 @@ static void liointc_resume(struct irq_chip_generic *gc) /* Restore map cache */ for (i = 0; i < LIOINTC_CHIP_IRQ; i++) writeb(priv->map_cache[i], gc->reg_base + i); + writel(priv->int_pol, gc->reg_base + LIOINTC_REG_INTC_POL); + writel(priv->int_edge, gc->reg_base + LIOINTC_REG_INTC_EDGE); /* Restore mask cache */ writel(gc->mask_cache, gc->reg_base + LIOINTC_REG_INTC_ENABLE); irq_gc_unlock_irqrestore(gc, flags); } -static const char * const parent_names[] = {"int0", "int1", "int2", "int3"}; +static int parent_irq[LIOINTC_NUM_PARENT]; +static u32 parent_int_map[LIOINTC_NUM_PARENT]; +static const char *const parent_names[] = {"int0", "int1", "int2", "int3"}; -int __init liointc_of_init(struct device_node *node, - struct device_node *parent) +static int liointc_init(phys_addr_t addr, unsigned long size, int revision, + struct fwnode_handle *domain_handle, struct device_node *node) { + int i, err; + void __iomem *base; + struct irq_chip_type *ct; struct irq_chip_generic *gc; struct irq_domain *domain; - struct irq_chip_type *ct; struct liointc_priv *priv; - void __iomem *base; - u32 of_parent_int_map[LIOINTC_NUM_PARENT]; - int parent_irq[LIOINTC_NUM_PARENT]; - bool have_parent = FALSE; - int sz, i, err = 0; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - base = of_iomap(node, 0); - if (!base) { - err = -ENODEV; + base = ioremap(addr, size); + if (!base) goto out_free_priv; - } - for (i = 0; i < LIOINTC_NUM_PARENT; i++) { - parent_irq[i] = of_irq_get_byname(node, parent_names[i]); - if (parent_irq[i] > 0) - have_parent = TRUE; - } - if (!have_parent) { - err = -ENODEV; - goto out_iounmap; - } - - sz = of_property_read_variable_u32_array(node, - "loongson,parent_int_map", - &of_parent_int_map[0], - LIOINTC_NUM_PARENT, - LIOINTC_NUM_PARENT); - if (sz < 4) { - pr_err("loongson-liointc: No parent_int_map\n"); - err = -ENODEV; - goto out_iounmap; - } + priv->domain_handle = domain_handle; for (i = 0; i < LIOINTC_NUM_PARENT; i++) - priv->handler[i].parent_int_map = of_parent_int_map[i]; + priv->handler[i].parent_int_map = parent_int_map[i]; /* Setup IRQ domain */ - domain = irq_domain_add_linear(node, 32, + domain = irq_domain_create_linear(domain_handle, LIOINTC_CHIP_IRQ, &irq_generic_chip_ops, priv); if (!domain) { pr_err("loongson-liointc: cannot add IRQ domain\n"); - err = -EINVAL; goto out_iounmap; } - err = irq_alloc_domain_generic_chips(domain, 32, 1, - node->full_name, handle_level_irq, - IRQ_NOPROBE, 0, 0); + err = irq_alloc_domain_generic_chips(domain, LIOINTC_CHIP_IRQ, 1, + (node ? node->full_name : "LIOINTC"), + handle_level_irq, 0, IRQ_NOPROBE, 0); if (err) { pr_err("loongson-liointc: unable to register IRQ domain\n"); goto out_free_domain; @@ -234,6 +225,7 @@ int __init liointc_of_init(struct device_node *node, gc->private = priv; gc->reg_base = base; gc->domain = domain; + gc->suspend = liointc_suspend; gc->resume = liointc_resume; ct = gc->chip_types; @@ -265,8 +257,90 @@ int __init liointc_of_init(struct device_node *node, out_free_priv: kfree(priv); - return err; + return -EINVAL; +} + +#ifdef CONFIG_OF + +static int __init liointc_of_init(struct device_node *node, + struct device_node *parent) +{ + bool have_parent = FALSE; + int sz, i, index, revision, err = 0; + struct resource res; + + if (!of_device_is_compatible(node, "loongson,liointc-2.0")) { + index = 0; + revision = 1; + } else { + index = of_property_match_string(node, "reg-names", "main"); + revision = 2; + } + + if (of_address_to_resource(node, index, &res)) + return -EINVAL; + + for (i = 0; i < LIOINTC_NUM_PARENT; i++) { + parent_irq[i] = of_irq_get_byname(node, parent_names[i]); + if (parent_irq[i] > 0) + have_parent = TRUE; + } + if (!have_parent) + return -ENODEV; + + sz = of_property_read_variable_u32_array(node, + "loongson,parent_int_map", + &parent_int_map[0], + LIOINTC_NUM_PARENT, + LIOINTC_NUM_PARENT); + if (sz < 4) { + pr_err("loongson-liointc: No parent_int_map\n"); + return -ENODEV; + } + + err = liointc_init(res.start, resource_size(&res), + revision, of_node_to_fwnode(node), node); + if (err < 0) + return err; + + return 0; } IRQCHIP_DECLARE(loongson_liointc_1_0, "loongson,liointc-1.0", liointc_of_init); IRQCHIP_DECLARE(loongson_liointc_1_0a, "loongson,liointc-1.0a", liointc_of_init); + +#endif + +#ifdef CONFIG_ACPI + +struct irq_domain *liointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lio_pic *acpi_liointc) +{ + int ret; + struct fwnode_handle *domain_handle; + + if (!acpi_liointc) + return NULL; + + parent_int_map[0] = acpi_liointc->cascade_map[0]; + parent_int_map[1] = acpi_liointc->cascade_map[1]; + + parent_irq[0] = irq_create_mapping(parent, acpi_liointc->cascade[0]); + if (!cpu_has_extioi) + parent_irq[1] = irq_create_mapping(parent, acpi_liointc->cascade[1]); + + domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_liointc); + if (!domain_handle) { + pr_err("Unable to allocate domain handle\n"); + return NULL; + } + + ret = liointc_init(acpi_liointc->address, acpi_liointc->size, + 1, domain_handle, NULL); + if (ret < 0) + return NULL; + + return irq_find_matching_fwnode(domain_handle, DOMAIN_BUS_ANY); +} + +#endif diff --git a/drivers/irqchip/irq-loongson-pch-lpc.c b/drivers/irqchip/irq-loongson-pch-lpc.c new file mode 100644 index 0000000000000000000000000000000000000000..c63f650951631a11dd809b4de5ab024fc557236a --- /dev/null +++ b/drivers/irqchip/irq-loongson-pch-lpc.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Jianmin Lv + * Loongson LPC support + */ + +#define pr_fmt(fmt) "lpc: " fmt + +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ +#define LPC_INT_CTL 0x00 +#define LPC_INT_ENA 0x04 +#define LPC_INT_STS 0x08 +#define LPC_INT_CLR 0x0c +#define LPC_INT_POL 0x10 +#define LPC_COUNT 16 + +struct pch_lpc { + void __iomem *base; + struct irq_domain *lpc_domain; + struct fwnode_handle *domain_handle; + raw_spinlock_t lpc_lock; + u32 saved_reg_ctl; + u32 saved_reg_ena; + u32 saved_reg_pol; +}; + +static struct pch_lpc *pch_lpc_priv; + +static void ack_lpc_irq(struct irq_data *d) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pch_lpc_priv->lpc_lock, flags); + writel(0x1 << d->irq, pch_lpc_priv->base + LPC_INT_CLR); + raw_spin_unlock_irqrestore(&pch_lpc_priv->lpc_lock, flags); +} +static void mask_lpc_irq(struct irq_data *d) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pch_lpc_priv->lpc_lock, flags); + writel(readl(pch_lpc_priv->base + LPC_INT_ENA) & (~(0x1 << (d->irq))), + pch_lpc_priv->base + LPC_INT_ENA); + raw_spin_unlock_irqrestore(&pch_lpc_priv->lpc_lock, flags); +} + +static void mask_ack_lpc_irq(struct irq_data *d) +{ +} + +static void unmask_lpc_irq(struct irq_data *d) +{ + unsigned long flags; + raw_spin_lock_irqsave(&pch_lpc_priv->lpc_lock, flags); + writel(readl(pch_lpc_priv->base + LPC_INT_ENA) | (0x1 << (d->irq)), + pch_lpc_priv->base + LPC_INT_ENA); + raw_spin_unlock_irqrestore(&pch_lpc_priv->lpc_lock, flags); +} + +static int lpc_set_type(struct irq_data *d, unsigned int type) +{ + u32 val; + u32 mask = 0x1 << (d->hwirq); + + if (!(type & IRQ_TYPE_LEVEL_MASK)) + return 0; + + val = readl(pch_lpc_priv->base + LPC_INT_POL); + + if (type == IRQ_TYPE_LEVEL_HIGH) + val |= mask; + else + val &= ~mask; + + writel(val, pch_lpc_priv->base + LPC_INT_POL); + + return 0; +} + +static struct irq_chip pch_lpc_irq_chip = { + .name = "PCH LPC", + .irq_mask = mask_lpc_irq, + .irq_unmask = unmask_lpc_irq, + .irq_ack = ack_lpc_irq, + .irq_mask_ack = mask_ack_lpc_irq, + .irq_eoi = unmask_lpc_irq, + .irq_set_type = lpc_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, +}; + +static void lpc_irq_dispatch(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + u32 pending; + + chained_irq_enter(chip, desc); + + pending = readl(pch_lpc_priv->base + LPC_INT_ENA); + pending &= readl(pch_lpc_priv->base + LPC_INT_STS); + if (!pending) + spurious_interrupt(); + + while (pending) { + int bit = __ffs(pending); + generic_handle_irq(bit); + pending &= ~BIT(bit); + } + chained_irq_exit(chip, desc); +} + +static int pch_lpc_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + irq_set_chip_and_handler(irq, &pch_lpc_irq_chip, handle_level_irq); + return 0; +} + +static const struct irq_domain_ops pch_lpc_domain_ops = { + .map = pch_lpc_map, + .translate = irq_domain_translate_twocell, +}; + +static void pch_lpc_reset(struct pch_lpc *priv) +{ + /* Enable the LPC interrupt, bit31: en bit30: edge */ + writel(0x80000000, priv->base + LPC_INT_CTL); + writel(0, priv->base + LPC_INT_ENA); + /* Clear all 18-bit interrpt bit */ + writel(0x3ffff, priv->base + LPC_INT_CLR); +} + +static int pch_lpc_disabled(struct pch_lpc *priv) +{ + return (readl(priv->base + LPC_INT_ENA) == 0xffffffff) && + (readl(priv->base + LPC_INT_STS) == 0xffffffff); +} + +static int pch_lpc_suspend(void) +{ + pch_lpc_priv->saved_reg_ctl = readl(pch_lpc_priv->base + LPC_INT_CTL); + pch_lpc_priv->saved_reg_ena = readl(pch_lpc_priv->base + LPC_INT_ENA); + pch_lpc_priv->saved_reg_pol = readl(pch_lpc_priv->base + LPC_INT_POL); + return 0; +} + +static void pch_lpc_resume(void) +{ + writel(pch_lpc_priv->saved_reg_ctl, pch_lpc_priv->base + LPC_INT_CTL); + writel(pch_lpc_priv->saved_reg_ena, pch_lpc_priv->base + LPC_INT_ENA); + writel(pch_lpc_priv->saved_reg_pol, pch_lpc_priv->base + LPC_INT_POL); +} + +static struct syscore_ops pch_lpc_syscore_ops = { + .suspend = pch_lpc_suspend, + .resume = pch_lpc_resume, +}; + +struct irq_domain *pch_lpc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lpc_pic *acpi_pchlpc) +{ + int parent_irq; + struct pch_lpc *priv; + struct irq_fwspec fwspec; + + if (!acpi_pchlpc) + return NULL; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; + + raw_spin_lock_init(&priv->lpc_lock); + + priv->base = ioremap(acpi_pchlpc->address, acpi_pchlpc->size); + if (!priv->base) { + goto free_priv; + } + + if (pch_lpc_disabled(priv)) { + pr_err("Failed to get LPC status\n"); + goto iounmap_base; + } + + priv->domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_pchlpc); + if (!priv->domain_handle) { + pr_err("Unable to allocate domain handle\n"); + goto iounmap_base; + } + priv->lpc_domain = irq_domain_add_legacy(NULL, LPC_COUNT, 0, 0, + &pch_lpc_domain_ops, priv); + if (!priv->lpc_domain) { + pr_err("Failed to create IRQ domain\n"); + goto iounmap_base; + } + pch_lpc_reset(priv); + + fwspec.fwnode = parent->fwnode; + fwspec.param[0] = acpi_pchlpc->cascade; + fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH; + fwspec.param_count = 2; + parent_irq = irq_create_fwspec_mapping(&fwspec); + irq_set_chained_handler_and_data(parent_irq, lpc_irq_dispatch, priv); + pch_lpc_priv = priv; + + register_syscore_ops(&pch_lpc_syscore_ops); + + return irq_find_matching_fwnode(priv->domain_handle, DOMAIN_BUS_ANY); + +iounmap_base: + iounmap(priv->base); +free_priv: + kfree(priv); + + return NULL; +} diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index 32562b7e681b53ade32eddb92c7bb6908c0bf6d6..a333dce8ec9c4c3ce14b8a4b388ac7f3c20e6979 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2020, Jiaxun Yang + * Jianmin Lv + * Huacai Chen * Loongson PCH MSI support */ @@ -15,14 +17,19 @@ #include #include +static int nr_pics; + struct pch_msi_data { struct mutex msi_map_lock; phys_addr_t doorbell; u32 irq_first; /* The vector number that MSIs starts */ u32 num_irqs; /* The number of vectors for MSIs */ unsigned long *msi_map; + struct fwnode_handle *domain_handle; }; +static struct pch_msi_data *pch_msi_priv[MAX_IO_PICS]; + static void pch_msi_mask_msi_irq(struct irq_data *d) { pci_msi_mask_irq(d); @@ -81,9 +88,36 @@ static void pch_msi_compose_msi_msg(struct irq_data *data, msg->data = data->hwirq; } +#define DEFAULT_MSI_LIMITS 256 + +static int pch_msi_limits = DEFAULT_MSI_LIMITS; + +static int __init pch_msi_limit(char *str) +{ + get_option(&str, &pch_msi_limits); + + if (pch_msi_limits <= 0) + pch_msi_limits = DEFAULT_MSI_LIMITS; + + return 0; +} + +early_param("loongson_msi_limit", pch_msi_limit); + +static int pch_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg) +{ + memset(arg, 0, sizeof(*arg)); + return clamp_val(nvec, 0, pch_msi_limits); +} + +static struct msi_domain_ops pch_msi_ops = { + .msi_prepare = pch_msi_prepare, +}; + static struct msi_domain_info pch_msi_domain_info = { .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &pch_msi_ops, .chip = &pch_msi_irq_chip, }; @@ -154,12 +188,14 @@ static const struct irq_domain_ops pch_msi_middle_domain_ops = { }; static int pch_msi_init_domains(struct pch_msi_data *priv, - struct device_node *node, - struct irq_domain *parent) + struct irq_domain *parent, + struct fwnode_handle *domain_handle) { struct irq_domain *middle_domain, *msi_domain; - middle_domain = irq_domain_create_linear(of_node_to_fwnode(node), + priv->domain_handle = domain_handle; + + middle_domain = irq_domain_create_linear(priv->domain_handle, priv->num_irqs, &pch_msi_middle_domain_ops, priv); @@ -171,7 +207,7 @@ static int pch_msi_init_domains(struct pch_msi_data *priv, middle_domain->parent = parent; irq_domain_update_bus_token(middle_domain, DOMAIN_BUS_NEXUS); - msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), + msi_domain = pci_msi_create_irq_domain(priv->domain_handle, &pch_msi_domain_info, middle_domain); if (!msi_domain) { @@ -183,19 +219,11 @@ static int pch_msi_init_domains(struct pch_msi_data *priv, return 0; } -static int pch_msi_init(struct device_node *node, - struct device_node *parent) +static int pch_msi_init(phys_addr_t msg_address, int irq_base, int irq_count, + struct irq_domain *parent_domain, struct fwnode_handle *domain_handle) { - struct pch_msi_data *priv; - struct irq_domain *parent_domain; - struct resource res; int ret; - - parent_domain = irq_find_host(parent); - if (!parent_domain) { - pr_err("Failed to find the parent domain\n"); - return -ENXIO; - } + struct pch_msi_data *priv; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -203,48 +231,93 @@ static int pch_msi_init(struct device_node *node, mutex_init(&priv->msi_map_lock); - ret = of_address_to_resource(node, 0, &res); - if (ret) { - pr_err("Failed to allocate resource\n"); - goto err_priv; - } - - priv->doorbell = res.start; - - if (of_property_read_u32(node, "loongson,msi-base-vec", - &priv->irq_first)) { - pr_err("Unable to parse MSI vec base\n"); - ret = -EINVAL; - goto err_priv; - } - - if (of_property_read_u32(node, "loongson,msi-num-vecs", - &priv->num_irqs)) { - pr_err("Unable to parse MSI vec number\n"); - ret = -EINVAL; - goto err_priv; - } + priv->doorbell = msg_address; + priv->irq_first = irq_base; + priv->num_irqs = irq_count; priv->msi_map = bitmap_zalloc(priv->num_irqs, GFP_KERNEL); - if (!priv->msi_map) { - ret = -ENOMEM; + if (!priv->msi_map) goto err_priv; - } pr_debug("Registering %d MSIs, starting at %d\n", priv->num_irqs, priv->irq_first); - ret = pch_msi_init_domains(priv, node, parent_domain); + ret = pch_msi_init_domains(priv, parent_domain, domain_handle); if (ret) goto err_map; + pch_msi_priv[nr_pics++] = priv; + return 0; err_map: kfree(priv->msi_map); err_priv: kfree(priv); - return ret; + + return -EINVAL; +} + +#ifdef CONFIG_OF + +static int pch_msi_of_init(struct device_node *node, struct device_node *parent) +{ + int err; + int irq_base, irq_count; + struct resource res; + struct irq_domain *parent_domain; + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + pr_err("Failed to find the parent domain\n"); + return -ENXIO; + } + + if (of_address_to_resource(node, 0, &res)) { + pr_err("Failed to allocate resource\n"); + return -EINVAL; + } + + if (of_property_read_u32(node, "loongson,msi-base-vec", &irq_base)) { + pr_err("Unable to parse MSI vec base\n"); + return -EINVAL; + } + + if (of_property_read_u32(node, "loongson,msi-num-vecs", &irq_count)) { + pr_err("Unable to parse MSI vec number\n"); + return -EINVAL; + } + + err = pch_msi_init(res.start, irq_base, irq_count, parent_domain, of_node_to_fwnode(node)); + if (err < 0) + return err; + + return 0; +} + +IRQCHIP_DECLARE(pch_msi, "loongson,pch-msi-1.0", pch_msi_of_init); + +#endif + +#ifdef CONFIG_ACPI + +struct irq_domain *pch_msi_acpi_init(struct irq_domain *parent, + struct acpi_madt_msi_pic *acpi_pchmsi) +{ + int ret; + struct fwnode_handle *domain_handle; + + if (!acpi_pchmsi) + return NULL; + + domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_pchmsi); + + ret = pch_msi_init(acpi_pchmsi->msg_address, acpi_pchmsi->start, + acpi_pchmsi->count, parent, domain_handle); + if (ret < 0) + return NULL; + + return irq_find_matching_fwnode(domain_handle, DOMAIN_BUS_PCI_MSI); } -IRQCHIP_DECLARE(pch_msi, "loongson,pch-msi-1.0", pch_msi_init); +#endif diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index a4eb8a2181c7f1c0b5c23bc8756c0588c34e4c6a..3d9b1c53ebd980ae7d4ed4f08c2d616f0a288a8d 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2020, Jiaxun Yang + * Jianmin Lv + * Huacai Chen * Loongson PCH PIC support */ @@ -15,6 +17,7 @@ #include #include #include +#include /* Registers */ #define PCH_PIC_MASK 0x20 @@ -32,14 +35,23 @@ #define PIC_COUNT (PIC_COUNT_PER_REG * PIC_REG_COUNT) #define PIC_REG_IDX(irq_id) ((irq_id) / PIC_COUNT_PER_REG) #define PIC_REG_BIT(irq_id) ((irq_id) % PIC_COUNT_PER_REG) +#define PCH_PIC_SIZE 0x400 + +static int nr_pics; struct pch_pic { void __iomem *base; struct irq_domain *pic_domain; + struct fwnode_handle *domain_handle; u32 ht_vec_base; raw_spinlock_t pic_lock; + u32 saved_vec_en[PIC_REG_COUNT]; + u32 saved_vec_pol[PIC_REG_COUNT]; + u32 saved_vec_edge[PIC_REG_COUNT]; }; +static struct pch_pic *pch_pic_priv[MAX_IO_PICS]; + static void pch_pic_bitset(struct pch_pic *priv, int offset, int bit) { u32 reg; @@ -137,6 +149,7 @@ static struct irq_chip pch_pic_irq_chip = { .irq_ack = pch_pic_ack_irq, .irq_set_affinity = irq_chip_set_affinity_parent, .irq_set_type = pch_pic_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE, }; static int pch_pic_alloc(struct irq_domain *domain, unsigned int virq, @@ -180,7 +193,7 @@ static void pch_pic_reset(struct pch_pic *priv) int i; for (i = 0; i < PIC_COUNT; i++) { - /* Write vectored ID */ + /* Write vector ID */ writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(i)); /* Hardcode route to HT0 Lo */ writeb(1, priv->base + PCH_INT_ROUTE(i)); @@ -198,50 +211,79 @@ static void pch_pic_reset(struct pch_pic *priv) } } -static int pch_pic_of_init(struct device_node *node, - struct device_node *parent) +static int pch_pic_suspend(void) +{ + int i, j; + + for (i = 0; i < nr_pics; i++) { + for (j = 0; j < PIC_REG_COUNT; j++) { + pch_pic_priv[i]->saved_vec_pol[j] = + readl(pch_pic_priv[i]->base + PCH_PIC_POL + 4 * j); + pch_pic_priv[i]->saved_vec_edge[j] = + readl(pch_pic_priv[i]->base + PCH_PIC_EDGE + 4 * j); + pch_pic_priv[i]->saved_vec_en[j] = + readl(pch_pic_priv[i]->base + PCH_PIC_MASK + 4 * j); + } + } + + return 0; +} + +static void pch_pic_resume(void) +{ + int i, j; + + for (i = 0; i < nr_pics; i++) { + pch_pic_reset(pch_pic_priv[i]); + for (j = 0; j < PIC_REG_COUNT; j++) { + writel(pch_pic_priv[i]->saved_vec_pol[j], + pch_pic_priv[i]->base + PCH_PIC_POL + 4 * j); + writel(pch_pic_priv[i]->saved_vec_edge[j], + pch_pic_priv[i]->base + PCH_PIC_EDGE + 4 * j); + writel(pch_pic_priv[i]->saved_vec_en[j], + pch_pic_priv[i]->base + PCH_PIC_MASK + 4 * j); + } + } +} + +static struct syscore_ops pch_pic_syscore_ops = { + .suspend = pch_pic_suspend, + .resume = pch_pic_resume, +}; + +static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, + struct irq_domain *parent_domain, struct fwnode_handle *domain_handle) { + int vec_count; struct pch_pic *priv; - struct irq_domain *parent_domain; - int err; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; raw_spin_lock_init(&priv->pic_lock); - priv->base = of_iomap(node, 0); - if (!priv->base) { - err = -ENOMEM; + priv->base = ioremap(addr, size); + if (!priv->base) goto free_priv; - } - parent_domain = irq_find_host(parent); - if (!parent_domain) { - pr_err("Failed to find the parent domain\n"); - err = -ENXIO; - goto iounmap_base; - } + priv->domain_handle = domain_handle; - if (of_property_read_u32(node, "loongson,pic-base-vec", - &priv->ht_vec_base)) { - pr_err("Failed to determine pic-base-vec\n"); - err = -EINVAL; - goto iounmap_base; - } + priv->ht_vec_base = vec_base; + vec_count = ((readq(priv->base) >> 48) & 0xff) + 1; priv->pic_domain = irq_domain_create_hierarchy(parent_domain, 0, - PIC_COUNT, - of_node_to_fwnode(node), - &pch_pic_domain_ops, - priv); + vec_count, priv->domain_handle, + &pch_pic_domain_ops, priv); + if (!priv->pic_domain) { pr_err("Failed to create IRQ domain\n"); - err = -ENOMEM; goto iounmap_base; } pch_pic_reset(priv); + pch_pic_priv[nr_pics++] = priv; + + register_syscore_ops(&pch_pic_syscore_ops); return 0; @@ -250,7 +292,69 @@ static int pch_pic_of_init(struct device_node *node, free_priv: kfree(priv); - return err; + return -EINVAL; +} + +#ifdef CONFIG_OF + +static int pch_pic_of_init(struct device_node *node, + struct device_node *parent) +{ + int err, vec_base; + struct resource res; + struct irq_domain *parent_domain; + + if (of_address_to_resource(node, 0, &res)) + return -EINVAL; + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + pr_err("Failed to find the parent domain\n"); + return -ENXIO; + } + + if (of_property_read_u32(node, "loongson,pic-base-vec", &vec_base)) { + pr_err("Failed to determine pic-base-vec\n"); + return -EINVAL; + } + + err = pch_pic_init(res.start, resource_size(&res), vec_base, + parent_domain, of_node_to_fwnode(node)); + if (err < 0) + return err; + + return 0; } IRQCHIP_DECLARE(pch_pic, "loongson,pch-pic-1.0", pch_pic_of_init); + +#endif + +#ifdef CONFIG_ACPI + +struct irq_domain *pch_pic_acpi_init(struct irq_domain *parent, + struct acpi_madt_bio_pic *acpi_pchpic) +{ + int ret, vec_base; + struct fwnode_handle *domain_handle; + + if (!acpi_pchpic) + return NULL; + + vec_base = acpi_pchpic->gsi_base - GSI_MIN_PCH_IRQ; + + domain_handle = irq_domain_alloc_fwnode((phys_addr_t *)acpi_pchpic); + if (!domain_handle) { + pr_err("Unable to allocate domain handle\n"); + return NULL; + } + + ret = pch_pic_init(acpi_pchpic->address, acpi_pchpic->size, + vec_base, parent, domain_handle); + if (ret < 0) + return NULL; + + return irq_find_matching_fwnode(domain_handle, DOMAIN_BUS_ANY); +} + +#endif diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig index ad84be4f6817144aa2080b46ca932e2c84f8f8e7..03c4c5e4d35cae05b33fa8bcc537795a5dacef04 100644 --- a/drivers/leds/Kconfig +++ b/drivers/leds/Kconfig @@ -116,6 +116,7 @@ config LEDS_AS3645A config LEDS_AW2013 tristate "LED support for Awinic AW2013" depends on LEDS_CLASS && I2C && OF + select REGMAP_I2C help This option enables support for the AW2013 3-channel LED driver. diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c index 467be48b9004c0f5b2a04cf92d7d948467ddc750..23a0c209744dc644849ee237702b6e42952624c9 100644 --- a/drivers/media/dvb-core/dvbdev.c +++ b/drivers/media/dvb-core/dvbdev.c @@ -114,6 +114,8 @@ static int dvb_device_open(struct inode *inode, struct file *file) err = file->f_op->open(inode, file); up_read(&minor_rwsem); mutex_unlock(&dvbdev_mutex); + if (err) + dvb_device_put(dvbdev); return err; } fail: diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index ff106d6ece68a3bc3d467dec8849e53c512792ec..94c6cc59372c65b42f01c90af93a54b98687bcb3 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1898,7 +1898,7 @@ static int m88ds3103_probe(struct i2c_client *client, /* get frontend address */ ret = regmap_read(dev->regmap, 0x29, &utmp); if (ret) - goto err_kfree; + goto err_del_adapters; dev->dt_addr = ((utmp & 0x80) == 0) ? 0x42 >> 1 : 0x40 >> 1; dev_dbg(&client->dev, "dt addr is 0x%02x\n", dev->dt_addr); @@ -1906,11 +1906,14 @@ static int m88ds3103_probe(struct i2c_client *client, dev->dt_addr); if (IS_ERR(dev->dt_client)) { ret = PTR_ERR(dev->dt_client); - goto err_kfree; + goto err_del_adapters; } } return 0; + +err_del_adapters: + i2c_mux_del_adapters(dev->muxc); err_kfree: kfree(dev); err: diff --git a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c index 36109c324cb6c9aeef7b21e66ac1d315be9984a7..3519c2252ae881af67fa7bd4a0fc585098f0fd58 100644 --- a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c +++ b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c @@ -977,13 +977,13 @@ static void mtk_jpeg_dec_device_run(void *priv) if (ret < 0) goto dec_end; - schedule_delayed_work(&jpeg->job_timeout_work, - msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC)); - mtk_jpeg_set_dec_src(ctx, &src_buf->vb2_buf, &bs); if (mtk_jpeg_set_dec_dst(ctx, &jpeg_src_buf->dec_param, &dst_buf->vb2_buf, &fb)) goto dec_end; + schedule_delayed_work(&jpeg->job_timeout_work, + msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC)); + spin_lock_irqsave(&jpeg->hw_lock, flags); mtk_jpeg_dec_reset(jpeg->reg_base); mtk_jpeg_dec_set_config(jpeg->reg_base, diff --git a/drivers/media/usb/cx231xx/cx231xx-core.c b/drivers/media/usb/cx231xx/cx231xx-core.c index 05d91caaed0cda183cd24925ece76f9a327b2981..46e34225b45c0bfab24aacc7067d4d9e5758033a 100644 --- a/drivers/media/usb/cx231xx/cx231xx-core.c +++ b/drivers/media/usb/cx231xx/cx231xx-core.c @@ -1024,6 +1024,7 @@ int cx231xx_init_isoc(struct cx231xx *dev, int max_packets, if (!dev->video_mode.isoc_ctl.urb) { dev_err(dev->dev, "cannot alloc memory for usb buffers\n"); + kfree(dma_q->p_left_data); return -ENOMEM; } @@ -1033,6 +1034,7 @@ int cx231xx_init_isoc(struct cx231xx *dev, int max_packets, dev_err(dev->dev, "cannot allocate memory for usbtransfer\n"); kfree(dev->video_mode.isoc_ctl.urb); + kfree(dma_q->p_left_data); return -ENOMEM; } diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index a5b2bf0e40cc16e493551e6285cb45425dc0024b..8fe4a0fd6ef181ccd46285e6830d68b029a4cfc9 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -1065,14 +1065,15 @@ config MMC_SDHCI_XENON config MMC_SDHCI_OMAP tristate "TI SDHCI Controller Support" + depends on ARCH_OMAP2PLUS || ARCH_KEYSTONE || COMPILE_TEST depends on MMC_SDHCI_PLTFM && OF select THERMAL imply TI_SOC_THERMAL select MMC_SDHCI_EXTERNAL_DMA if DMA_ENGINE help This selects the Secure Digital Host Controller Interface (SDHCI) - support present in TI's DRA7 SOCs. The controller supports - SD/MMC/SDIO devices. + support present in TI's Keystone/OMAP2+/DRA7 SOCs. The controller + supports SD/MMC/SDIO devices. If you have a controller with this interface, say Y or M here. @@ -1080,14 +1081,15 @@ config MMC_SDHCI_OMAP config MMC_SDHCI_AM654 tristate "Support for the SDHCI Controller in TI's AM654 SOCs" + depends on ARCH_K3 || COMPILE_TEST depends on MMC_SDHCI_PLTFM && OF select MMC_SDHCI_IO_ACCESSORS select MMC_CQHCI select REGMAP_MMIO help This selects the Secure Digital Host Controller Interface (SDHCI) - support present in TI's AM654 SOCs. The controller supports - SD/MMC/SDIO devices. + support present in TI's AM65x/AM64x/AM62x/J721E SOCs. The controller + supports SD/MMC/SDIO devices. If you have a controller with this interface, say Y or M here. diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c index 1d814919eb6be435f109b04fc415871467101bac..a1fb5d0e9553a433116417c20152e354b6c370b8 100644 --- a/drivers/mmc/host/mmc_spi.c +++ b/drivers/mmc/host/mmc_spi.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -119,19 +119,14 @@ struct mmc_spi_host { struct spi_transfer status; struct spi_message readback; - /* underlying DMA-aware controller, or null */ - struct device *dma_dev; - /* buffer used for commands and for message "overhead" */ struct scratch *data; - dma_addr_t data_dma; /* Specs say to write ones most of the time, even when the card * has no need to read its input data; and many cards won't care. * This is our source of those ones. */ void *ones; - dma_addr_t ones_dma; }; @@ -147,11 +142,8 @@ static inline int mmc_cs_off(struct mmc_spi_host *host) return spi_setup(host->spi); } -static int -mmc_spi_readbytes(struct mmc_spi_host *host, unsigned len) +static int mmc_spi_readbytes(struct mmc_spi_host *host, unsigned int len) { - int status; - if (len > sizeof(*host->data)) { WARN_ON(1); return -EIO; @@ -159,19 +151,7 @@ mmc_spi_readbytes(struct mmc_spi_host *host, unsigned len) host->status.len = len; - if (host->dma_dev) - dma_sync_single_for_device(host->dma_dev, - host->data_dma, sizeof(*host->data), - DMA_FROM_DEVICE); - - status = spi_sync_locked(host->spi, &host->readback); - - if (host->dma_dev) - dma_sync_single_for_cpu(host->dma_dev, - host->data_dma, sizeof(*host->data), - DMA_FROM_DEVICE); - - return status; + return spi_sync_locked(host->spi, &host->readback); } static int mmc_spi_skip(struct mmc_spi_host *host, unsigned long timeout, @@ -513,23 +493,11 @@ mmc_spi_command_send(struct mmc_spi_host *host, t = &host->t; memset(t, 0, sizeof(*t)); t->tx_buf = t->rx_buf = data->status; - t->tx_dma = t->rx_dma = host->data_dma; t->len = cp - data->status; t->cs_change = 1; spi_message_add_tail(t, &host->m); - if (host->dma_dev) { - host->m.is_dma_mapped = 1; - dma_sync_single_for_device(host->dma_dev, - host->data_dma, sizeof(*host->data), - DMA_BIDIRECTIONAL); - } status = spi_sync_locked(host->spi, &host->m); - - if (host->dma_dev) - dma_sync_single_for_cpu(host->dma_dev, - host->data_dma, sizeof(*host->data), - DMA_BIDIRECTIONAL); if (status < 0) { dev_dbg(&host->spi->dev, " ... write returned %d\n", status); cmd->error = status; @@ -547,9 +515,6 @@ mmc_spi_command_send(struct mmc_spi_host *host, * We always provide TX data for data and CRC. The MMC/SD protocol * requires us to write ones; but Linux defaults to writing zeroes; * so we explicitly initialize it to all ones on RX paths. - * - * We also handle DMA mapping, so the underlying SPI controller does - * not need to (re)do it for each message. */ static void mmc_spi_setup_data_message( @@ -559,11 +524,8 @@ mmc_spi_setup_data_message( { struct spi_transfer *t; struct scratch *scratch = host->data; - dma_addr_t dma = host->data_dma; spi_message_init(&host->m); - if (dma) - host->m.is_dma_mapped = 1; /* for reads, readblock() skips 0xff bytes before finding * the token; for writes, this transfer issues that token. @@ -577,8 +539,6 @@ mmc_spi_setup_data_message( else scratch->data_token = SPI_TOKEN_SINGLE; t->tx_buf = &scratch->data_token; - if (dma) - t->tx_dma = dma + offsetof(struct scratch, data_token); spi_message_add_tail(t, &host->m); } @@ -588,7 +548,6 @@ mmc_spi_setup_data_message( t = &host->t; memset(t, 0, sizeof(*t)); t->tx_buf = host->ones; - t->tx_dma = host->ones_dma; /* length and actual buffer info are written later */ spi_message_add_tail(t, &host->m); @@ -598,14 +557,9 @@ mmc_spi_setup_data_message( if (direction == DMA_TO_DEVICE) { /* the actual CRC may get written later */ t->tx_buf = &scratch->crc_val; - if (dma) - t->tx_dma = dma + offsetof(struct scratch, crc_val); } else { t->tx_buf = host->ones; - t->tx_dma = host->ones_dma; t->rx_buf = &scratch->crc_val; - if (dma) - t->rx_dma = dma + offsetof(struct scratch, crc_val); } spi_message_add_tail(t, &host->m); @@ -628,10 +582,7 @@ mmc_spi_setup_data_message( memset(t, 0, sizeof(*t)); t->len = (direction == DMA_TO_DEVICE) ? sizeof(scratch->status) : 1; t->tx_buf = host->ones; - t->tx_dma = host->ones_dma; t->rx_buf = scratch->status; - if (dma) - t->rx_dma = dma + offsetof(struct scratch, status); t->cs_change = 1; spi_message_add_tail(t, &host->m); } @@ -660,23 +611,13 @@ mmc_spi_writeblock(struct mmc_spi_host *host, struct spi_transfer *t, if (host->mmc->use_spi_crc) scratch->crc_val = cpu_to_be16(crc_itu_t(0, t->tx_buf, t->len)); - if (host->dma_dev) - dma_sync_single_for_device(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); status = spi_sync_locked(spi, &host->m); - if (status != 0) { dev_dbg(&spi->dev, "write error (%d)\n", status); return status; } - if (host->dma_dev) - dma_sync_single_for_cpu(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); - /* * Get the transmission data-response reply. It must follow * immediately after the data block we transferred. This reply @@ -725,8 +666,6 @@ mmc_spi_writeblock(struct mmc_spi_host *host, struct spi_transfer *t, } t->tx_buf += t->len; - if (host->dma_dev) - t->tx_dma += t->len; /* Return when not busy. If we didn't collect that status yet, * we'll need some more I/O. @@ -790,30 +729,12 @@ mmc_spi_readblock(struct mmc_spi_host *host, struct spi_transfer *t, } leftover = status << 1; - if (host->dma_dev) { - dma_sync_single_for_device(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); - dma_sync_single_for_device(host->dma_dev, - t->rx_dma, t->len, - DMA_FROM_DEVICE); - } - status = spi_sync_locked(spi, &host->m); if (status < 0) { dev_dbg(&spi->dev, "read error %d\n", status); return status; } - if (host->dma_dev) { - dma_sync_single_for_cpu(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); - dma_sync_single_for_cpu(host->dma_dev, - t->rx_dma, t->len, - DMA_FROM_DEVICE); - } - if (bitshift) { /* Walk through the data and the crc and do * all the magic to get byte-aligned data. @@ -848,8 +769,6 @@ mmc_spi_readblock(struct mmc_spi_host *host, struct spi_transfer *t, } t->rx_buf += t->len; - if (host->dma_dev) - t->rx_dma += t->len; return 0; } @@ -864,7 +783,6 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, struct mmc_data *data, u32 blk_size) { struct spi_device *spi = host->spi; - struct device *dma_dev = host->dma_dev; struct spi_transfer *t; enum dma_data_direction direction; struct scatterlist *sg; @@ -891,31 +809,8 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, */ for_each_sg(data->sg, sg, data->sg_len, n_sg) { int status = 0; - dma_addr_t dma_addr = 0; void *kmap_addr; unsigned length = sg->length; - enum dma_data_direction dir = direction; - - /* set up dma mapping for controller drivers that might - * use DMA ... though they may fall back to PIO - */ - if (dma_dev) { - /* never invalidate whole *shared* pages ... */ - if ((sg->offset != 0 || length != PAGE_SIZE) - && dir == DMA_FROM_DEVICE) - dir = DMA_BIDIRECTIONAL; - - dma_addr = dma_map_page(dma_dev, sg_page(sg), 0, - PAGE_SIZE, dir); - if (dma_mapping_error(dma_dev, dma_addr)) { - data->error = -EFAULT; - break; - } - if (direction == DMA_TO_DEVICE) - t->tx_dma = dma_addr + sg->offset; - else - t->rx_dma = dma_addr + sg->offset; - } /* allow pio too; we don't allow highmem */ kmap_addr = kmap(sg_page(sg)); @@ -951,8 +846,6 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, if (direction == DMA_FROM_DEVICE) flush_kernel_dcache_page(sg_page(sg)); kunmap(sg_page(sg)); - if (dma_dev) - dma_unmap_page(dma_dev, dma_addr, PAGE_SIZE, dir); if (status < 0) { data->error = status; @@ -989,21 +882,9 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, scratch->status[0] = SPI_TOKEN_STOP_TRAN; host->early_status.tx_buf = host->early_status.rx_buf; - host->early_status.tx_dma = host->early_status.rx_dma; host->early_status.len = statlen; - if (host->dma_dev) - dma_sync_single_for_device(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); - tmp = spi_sync_locked(spi, &host->m); - - if (host->dma_dev) - dma_sync_single_for_cpu(host->dma_dev, - host->data_dma, sizeof(*scratch), - DMA_BIDIRECTIONAL); - if (tmp < 0) { if (!data->error) data->error = tmp; @@ -1278,52 +1159,6 @@ mmc_spi_detect_irq(int irq, void *mmc) return IRQ_HANDLED; } -#ifdef CONFIG_HAS_DMA -static int mmc_spi_dma_alloc(struct mmc_spi_host *host) -{ - struct spi_device *spi = host->spi; - struct device *dev; - - if (!spi->master->dev.parent->dma_mask) - return 0; - - dev = spi->master->dev.parent; - - host->ones_dma = dma_map_single(dev, host->ones, MMC_SPI_BLOCKSIZE, - DMA_TO_DEVICE); - if (dma_mapping_error(dev, host->ones_dma)) - return -ENOMEM; - - host->data_dma = dma_map_single(dev, host->data, sizeof(*host->data), - DMA_BIDIRECTIONAL); - if (dma_mapping_error(dev, host->data_dma)) { - dma_unmap_single(dev, host->ones_dma, MMC_SPI_BLOCKSIZE, - DMA_TO_DEVICE); - return -ENOMEM; - } - - dma_sync_single_for_cpu(dev, host->data_dma, sizeof(*host->data), - DMA_BIDIRECTIONAL); - - host->dma_dev = dev; - return 0; -} - -static void mmc_spi_dma_free(struct mmc_spi_host *host) -{ - if (!host->dma_dev) - return; - - dma_unmap_single(host->dma_dev, host->ones_dma, MMC_SPI_BLOCKSIZE, - DMA_TO_DEVICE); - dma_unmap_single(host->dma_dev, host->data_dma, sizeof(*host->data), - DMA_BIDIRECTIONAL); -} -#else -static inline int mmc_spi_dma_alloc(struct mmc_spi_host *host) { return 0; } -static inline void mmc_spi_dma_free(struct mmc_spi_host *host) {} -#endif - static int mmc_spi_probe(struct spi_device *spi) { void *ones; @@ -1415,24 +1250,17 @@ static int mmc_spi_probe(struct spi_device *spi) dev_set_drvdata(&spi->dev, mmc); - /* preallocate dma buffers */ + /* Preallocate buffers */ host->data = kmalloc(sizeof(*host->data), GFP_KERNEL); if (!host->data) goto fail_nobuf1; - status = mmc_spi_dma_alloc(host); - if (status) - goto fail_dma; - /* setup message for status/busy readback */ spi_message_init(&host->readback); - host->readback.is_dma_mapped = (host->dma_dev != NULL); spi_message_add_tail(&host->status, &host->readback); host->status.tx_buf = host->ones; - host->status.tx_dma = host->ones_dma; host->status.rx_buf = &host->data->status; - host->status.rx_dma = host->data_dma + offsetof(struct scratch, status); host->status.cs_change = 1; /* register card detect irq */ @@ -1477,9 +1305,8 @@ static int mmc_spi_probe(struct spi_device *spi) if (!status) has_ro = true; - dev_info(&spi->dev, "SD/MMC host %s%s%s%s%s\n", + dev_info(&spi->dev, "SD/MMC host %s%s%s%s\n", dev_name(&mmc->class_dev), - host->dma_dev ? "" : ", no DMA", has_ro ? "" : ", no WP", (host->pdata && host->pdata->setpower) ? "" : ", no poweroff", @@ -1490,8 +1317,6 @@ static int mmc_spi_probe(struct spi_device *spi) fail_gpiod_request: mmc_remove_host(mmc); fail_glue_init: - mmc_spi_dma_free(host); -fail_dma: kfree(host->data); fail_nobuf1: mmc_free_host(mmc); @@ -1513,7 +1338,6 @@ static int mmc_spi_remove(struct spi_device *spi) mmc_remove_host(mmc); - mmc_spi_dma_free(host); kfree(host->data); kfree(host->ones); diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c index 64ba465741a7896ef77827ff6a930671e0183977..81a5e7622ea7d6a5b7150ce27a01ffd349c50a0d 100644 --- a/drivers/net/bonding/bond_alb.c +++ b/drivers/net/bonding/bond_alb.c @@ -971,7 +971,8 @@ static int alb_upper_dev_walk(struct net_device *upper, if (netif_is_macvlan(upper) && !strict_match) { tags = bond_verify_device_path(bond->dev, upper, 0); if (IS_ERR_OR_NULL(tags)) - BUG(); + return -ENOMEM; + alb_send_lp_vid(slave, upper->dev_addr, tags[0].vlan_proto, tags[0].vlan_id); kfree(tags); diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 50fabba04248887b44dd7d74e619d9eb3b395684..506d6fdbfacc4ab1460e4912dcb9e5bcc9d3fd72 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -2777,8 +2777,11 @@ struct bond_vlan_tag *bond_verify_device_path(struct net_device *start_dev, if (start_dev == end_dev) { tags = kcalloc(level + 1, sizeof(*tags), GFP_ATOMIC); - if (!tags) + if (!tags) { + net_err_ratelimited("%s: %s: Failed to allocate tags\n", + __func__, start_dev->name); return ERR_PTR(-ENOMEM); + } tags[level].vlan_proto = VLAN_N_VID; return tags; } diff --git a/drivers/net/dsa/vitesse-vsc73xx-core.c b/drivers/net/dsa/vitesse-vsc73xx-core.c index 80eadf509c0a940fafc6aafc8a3993108277c9f0..018988b95035ebca9d579293ac82c52a9016a1fa 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-core.c +++ b/drivers/net/dsa/vitesse-vsc73xx-core.c @@ -1119,6 +1119,8 @@ static int vsc73xx_gpio_probe(struct vsc73xx *vsc) vsc->gc.label = devm_kasprintf(vsc->dev, GFP_KERNEL, "VSC%04x", vsc->chipid); + if (!vsc->gc.label) + return -ENOMEM; vsc->gc.ngpio = 4; vsc->gc.owner = THIS_MODULE; vsc->gc.parent = vsc->dev; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 584f365de563f0b473101c4897e7aeff25588c1e..059552f4154d13d7742d4aa92d93b01e7caad235 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -11331,6 +11331,11 @@ static int bnxt_fw_init_one_p1(struct bnxt *bp) bp->fw_cap = 0; rc = bnxt_hwrm_ver_get(bp); + /* FW may be unresponsive after FLR. FLR must complete within 100 msec + * so wait before continuing with recovery. + */ + if (rc) + msleep(100); bnxt_try_map_fw_health_reg(bp); if (rc) { if (bp->fw_health && bp->fw_health->status_reliable) { diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 4ce913559c91d8206b978047683aa3e106bc8f56..fe29769cb158993f8d2bb9d6b073d17d24de4487 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -1766,6 +1766,7 @@ static void fec_enet_adjust_link(struct net_device *ndev) /* if any of the above changed restart the FEC */ if (status_change) { + netif_stop_queue(ndev); napi_disable(&fep->napi); netif_tx_lock_bh(ndev); fec_restart(ndev); @@ -1775,6 +1776,7 @@ static void fec_enet_adjust_link(struct net_device *ndev) } } else { if (fep->link) { + netif_stop_queue(ndev); napi_disable(&fep->napi); netif_tx_lock_bh(ndev); fec_stop(ndev); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index e0e6275b3e20cb8decc42f23c80fd56365461fc0..e4e80c2b1ce400b3840e8cf762410f8eea9db85e 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -581,12 +581,38 @@ static int mvpp2_bm_pools_init(struct device *dev, struct mvpp2 *priv) return err; } +/* Cleanup pool before actual initialization in the OS */ +static void mvpp2_bm_pool_cleanup(struct mvpp2 *priv, int pool_id) +{ + unsigned int thread = mvpp2_cpu_to_thread(priv, get_cpu()); + u32 val; + int i; + + /* Drain the BM from all possible residues left by firmware */ + for (i = 0; i < MVPP2_BM_POOL_SIZE_MAX; i++) + mvpp2_thread_read(priv, thread, MVPP2_BM_PHY_ALLOC_REG(pool_id)); + + put_cpu(); + + /* Stop the BM pool */ + val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(pool_id)); + val |= MVPP2_BM_STOP_MASK; + mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(pool_id), val); +} + static int mvpp2_bm_init(struct device *dev, struct mvpp2 *priv) { enum dma_data_direction dma_dir = DMA_FROM_DEVICE; int i, err, poolnum = MVPP2_BM_POOLS_NUM; struct mvpp2_port *port; + if (priv->percpu_pools) + poolnum = mvpp2_get_nrxqs(priv) * 2; + + /* Clean up the pool state in case it contains stale state */ + for (i = 0; i < poolnum; i++) + mvpp2_bm_pool_cleanup(priv, i); + if (priv->percpu_pools) { for (i = 0; i < priv->port_count; i++) { port = priv->port_list[i]; @@ -596,7 +622,6 @@ static int mvpp2_bm_init(struct device *dev, struct mvpp2 *priv) } } - poolnum = mvpp2_get_nrxqs(priv) * 2; for (i = 0; i < poolnum; i++) { /* the pool in use */ int pn = i / (poolnum / 2); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index aa9e616cc1d59313375bff4f934c30e982b7ce84..011210e6842de7e7ca69ee2eb27167f53f84079a 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -2302,7 +2302,6 @@ static int mtk_open(struct net_device *dev) if (!refcount_read(ð->dma_refcnt)) { int err = mtk_start_dma(eth); - if (err) if (err) { phylink_disconnect_phy(mac->phylink); return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 39475f6565c73dd5cb6cccf769e5df8f4f19987e..7c436bdcf5b5fb6580c553713a883f641d435c38 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -208,11 +208,13 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); - in = kvzalloc(inlen, GFP_KERNEL); - if (!in || !ft->g) { - kfree(ft->g); - kvfree(in); + if (!ft->g) return -ENOMEM; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_free_g; } mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); @@ -232,7 +234,7 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, break; default: err = -EINVAL; - goto out; + goto err_free_in; } switch (type) { @@ -254,7 +256,7 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, break; default: err = -EINVAL; - goto out; + goto err_free_in; } MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); @@ -263,7 +265,7 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) - goto err; + goto err_clean_group; ft->num_groups++; memset(in, 0, inlen); @@ -272,18 +274,20 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) - goto err; + goto err_clean_group; ft->num_groups++; kvfree(in); return 0; -err: +err_clean_group: err = PTR_ERR(ft->g[ft->num_groups]); ft->g[ft->num_groups] = NULL; -out: +err_free_in: kvfree(in); - +err_free_g: + kfree(ft->g); + ft->g = NULL; return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index df1363a34a429cfffb245e7e868f6e057b80bfb0..9721fe58eb7b0fb50b41515d269d68d452eddd97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -667,6 +667,7 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, switch (action_type) { case DR_ACTION_TYP_DROP: attr.final_icm_addr = nic_dmn->drop_icm_addr; + attr.hit_gvmi = nic_dmn->drop_icm_addr >> 48; break; case DR_ACTION_TYP_FT: dest_action = action; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c index ded4cf6586809b8055dc69c1740462062aaf49e0..4b713832fdd5591f627bab82128923796ad57222 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c @@ -119,7 +119,6 @@ mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion) { struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; struct mlxsw_sp_acl_atcam_region_12kb *region_12kb; - size_t alloc_size; u64 max_lkey_id; int err; @@ -131,8 +130,7 @@ mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion) if (!region_12kb) return -ENOMEM; - alloc_size = BITS_TO_LONGS(max_lkey_id) * sizeof(unsigned long); - region_12kb->used_lkey_id = kzalloc(alloc_size, GFP_KERNEL); + region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL); if (!region_12kb->used_lkey_id) { err = -ENOMEM; goto err_used_lkey_id_alloc; @@ -149,7 +147,7 @@ mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion) return 0; err_rhashtable_init: - kfree(region_12kb->used_lkey_id); + bitmap_free(region_12kb->used_lkey_id); err_used_lkey_id_alloc: kfree(region_12kb); return err; @@ -161,7 +159,7 @@ mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion) struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; rhashtable_destroy(®ion_12kb->lkey_ht); - kfree(region_12kb->used_lkey_id); + bitmap_free(region_12kb->used_lkey_id); kfree(region_12kb); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c index 4c98950380d536ed30d146b3a6eeac9b098a1fae..d231f4d2888beefe838fadf6933e05ade826fdeb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c @@ -301,6 +301,7 @@ mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, unsigned long *p_index) { unsigned int num_rows, entry_size; + unsigned long index; /* We only allow allocations of entire rows */ if (num_erps % erp_core->num_erp_banks != 0) @@ -309,10 +310,11 @@ mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, entry_size = erp_core->erpt_entries_size[region_type]; num_rows = num_erps / erp_core->num_erp_banks; - *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); - if (*p_index == 0) + index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); + if (!index) return -ENOBUFS; - *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; + + *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c index 7cccc41dd69c9cb4a7cba4c226b143ff5bb00bc4..483c8b75bebb8d99726105e248b609f30cd9cbe8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c @@ -29,70 +29,6 @@ size_t mlxsw_sp_acl_tcam_priv_size(struct mlxsw_sp *mlxsw_sp) #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN 3000 /* ms */ #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS 100 /* number of entries */ -int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - u64 max_tcam_regions; - u64 max_regions; - u64 max_groups; - size_t alloc_size; - int err; - - mutex_init(&tcam->lock); - tcam->vregion_rehash_intrvl = - MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT; - INIT_LIST_HEAD(&tcam->vregion_list); - - max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, - ACL_MAX_TCAM_REGIONS); - max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); - - /* Use 1:1 mapping between ACL region and TCAM region */ - if (max_tcam_regions < max_regions) - max_regions = max_tcam_regions; - - alloc_size = sizeof(tcam->used_regions[0]) * BITS_TO_LONGS(max_regions); - tcam->used_regions = kzalloc(alloc_size, GFP_KERNEL); - if (!tcam->used_regions) - return -ENOMEM; - tcam->max_regions = max_regions; - - max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS); - alloc_size = sizeof(tcam->used_groups[0]) * BITS_TO_LONGS(max_groups); - tcam->used_groups = kzalloc(alloc_size, GFP_KERNEL); - if (!tcam->used_groups) { - err = -ENOMEM; - goto err_alloc_used_groups; - } - tcam->max_groups = max_groups; - tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, - ACL_MAX_GROUP_SIZE); - - err = ops->init(mlxsw_sp, tcam->priv, tcam); - if (err) - goto err_tcam_init; - - return 0; - -err_tcam_init: - kfree(tcam->used_groups); -err_alloc_used_groups: - kfree(tcam->used_regions); - return err; -} - -void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - - mutex_destroy(&tcam->lock); - ops->fini(mlxsw_sp, tcam->priv); - kfree(tcam->used_groups); - kfree(tcam->used_regions); -} - int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, u32 *priority, bool fillup_priority) @@ -1545,6 +1481,73 @@ mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_acl_tcam_vregion_rehash_end(mlxsw_sp, vregion, ctx); } +int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_tcam *tcam) +{ + const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; + u64 max_tcam_regions; + u64 max_regions; + u64 max_groups; + int err; + + mutex_init(&tcam->lock); + tcam->vregion_rehash_intrvl = + MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT; + INIT_LIST_HEAD(&tcam->vregion_list); + + max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, + ACL_MAX_TCAM_REGIONS); + max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); + + /* Use 1:1 mapping between ACL region and TCAM region */ + if (max_tcam_regions < max_regions) + max_regions = max_tcam_regions; + + tcam->used_regions = bitmap_zalloc(max_regions, GFP_KERNEL); + if (!tcam->used_regions) { + err = -ENOMEM; + goto err_alloc_used_regions; + } + tcam->max_regions = max_regions; + + max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS); + tcam->used_groups = bitmap_zalloc(max_groups, GFP_KERNEL); + if (!tcam->used_groups) { + err = -ENOMEM; + goto err_alloc_used_groups; + } + tcam->max_groups = max_groups; + tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, + ACL_MAX_GROUP_SIZE); + tcam->max_group_size = min_t(unsigned int, tcam->max_group_size, + MLXSW_REG_PAGT_ACL_MAX_NUM); + + err = ops->init(mlxsw_sp, tcam->priv, tcam); + if (err) + goto err_tcam_init; + + return 0; + +err_tcam_init: + bitmap_free(tcam->used_groups); +err_alloc_used_groups: + bitmap_free(tcam->used_regions); +err_alloc_used_regions: + mutex_destroy(&tcam->lock); + return err; +} + +void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_tcam *tcam) +{ + const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; + + ops->fini(mlxsw_sp, tcam->priv); + bitmap_free(tcam->used_groups); + bitmap_free(tcam->used_regions); + mutex_destroy(&tcam->lock); +} + static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = { MLXSW_AFK_ELEMENT_SRC_SYS_PORT, MLXSW_AFK_ELEMENT_DMAC_32_47, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c index b65b93a2b9bc607022e36a01998d1896b3c54798..fc2257753b9b39d2e9a97c06cf8df8b3230639a3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c @@ -122,7 +122,6 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) unsigned int sub_pools_count = ARRAY_SIZE(mlxsw_sp_counter_sub_pools); struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); struct mlxsw_sp_counter_pool *pool; - unsigned int map_size; int err; pool = kzalloc(struct_size(pool, sub_pools, sub_pools_count), @@ -143,9 +142,7 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) devlink_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_COUNTERS, mlxsw_sp_counter_pool_occ_get, pool); - map_size = BITS_TO_LONGS(pool->pool_size) * sizeof(unsigned long); - - pool->usage = kzalloc(map_size, GFP_KERNEL); + pool->usage = bitmap_zalloc(pool->pool_size, GFP_KERNEL); if (!pool->usage) { err = -ENOMEM; goto err_usage_alloc; @@ -158,7 +155,7 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) return 0; err_sub_pools_init: - kfree(pool->usage); + bitmap_free(pool->usage); err_usage_alloc: devlink_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_COUNTERS); @@ -176,7 +173,7 @@ void mlxsw_sp_counter_pool_fini(struct mlxsw_sp *mlxsw_sp) WARN_ON(find_first_bit(pool->usage, pool->pool_size) != pool->pool_size); WARN_ON(atomic_read(&pool->active_entries_count)); - kfree(pool->usage); + bitmap_free(pool->usage); devlink_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_COUNTERS); kfree(pool); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c index 368fa0e5ad3152e150e983ac632bb76cc78a1b17..ea37f5000caa1fee5c9958857d5fe8e4da5a517e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c @@ -1631,16 +1631,13 @@ mlxsw_sp_mid *__mlxsw_sp_mc_alloc(struct mlxsw_sp *mlxsw_sp, u16 fid) { struct mlxsw_sp_mid *mid; - size_t alloc_size; mid = kzalloc(sizeof(*mid), GFP_KERNEL); if (!mid) return NULL; - alloc_size = sizeof(unsigned long) * - BITS_TO_LONGS(mlxsw_core_max_ports(mlxsw_sp->core)); - - mid->ports_in_mid = kzalloc(alloc_size, GFP_KERNEL); + mid->ports_in_mid = bitmap_zalloc(mlxsw_core_max_ports(mlxsw_sp->core), + GFP_KERNEL); if (!mid->ports_in_mid) goto err_ports_in_mid_alloc; @@ -1659,7 +1656,7 @@ mlxsw_sp_mid *__mlxsw_sp_mc_alloc(struct mlxsw_sp *mlxsw_sp, return mid; err_write_mdb_entry: - kfree(mid->ports_in_mid); + bitmap_free(mid->ports_in_mid); err_ports_in_mid_alloc: kfree(mid); return NULL; @@ -1676,7 +1673,7 @@ static int mlxsw_sp_port_remove_from_mid(struct mlxsw_sp_port *mlxsw_sp_port, mlxsw_core_max_ports(mlxsw_sp->core))) { err = mlxsw_sp_mc_remove_mdb_entry(mlxsw_sp, mid); list_del(&mid->list); - kfree(mid->ports_in_mid); + bitmap_free(mid->ports_in_mid); kfree(mid); } return err; diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c index 8d51b0cb545ca6edeb96505c2665360863982a91..93fa10ad08a0cfd09a4b9d7234f1b6b1d5b78484 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c @@ -389,7 +389,7 @@ static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev) struct rtnl_link_ops rmnet_link_ops __read_mostly = { .kind = "rmnet", - .maxtype = __IFLA_RMNET_MAX, + .maxtype = IFLA_RMNET_MAX, .priv_size = sizeof(struct rmnet_priv), .setup = rmnet_vnd_setup, .validate = rmnet_rtnl_validate, diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index f092f468016bdd3eb3e6050f50f5ad3eafd23794..8a4dff0566f7d0eb07c1a7c7730d318a30de66a4 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1500,7 +1500,7 @@ static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev) struct ravb_tstamp_skb *ts_skb; struct ravb_tx_desc *desc; unsigned long flags; - u32 dma_addr; + dma_addr_t dma_addr; void *buffer; u32 entry; u32 len; diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 53f14c5a9e02c58b53279741f2a69da352ca7557..13eda0c37f7c59c614614ba0f3729ef236f0ae1d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -230,6 +230,15 @@ config DWMAC_INTEL This selects the Intel platform specific bus support for the stmmac driver. This driver is used for Intel Quark/EHL/TGL. +config DWMAC_LOONGSON + tristate "Loongson PCI DWMAC support" + default MACH_LOONGSON64 + depends on STMMAC_ETH && PCI + depends on COMMON_CLK + help + This selects the LOONGSON PCI bus support for the stmmac driver, + Support for ethernet controller on Loongson-2K1000 SoC and LS7A1000 bridge. + config STMMAC_PCI tristate "STMMAC PCI bus support" depends on STMMAC_ETH && PCI diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index 24e6145d4eae50822359ef9b1c138ab3a39ba170..11ea4569c43d92ab4c74b7cb061e3a4258d74920 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -34,4 +34,5 @@ dwmac-altr-socfpga-objs := altr_tse_pcs.o dwmac-socfpga.o obj-$(CONFIG_STMMAC_PCI) += stmmac-pci.o obj-$(CONFIG_DWMAC_INTEL) += dwmac-intel.o +obj-$(CONFIG_DWMAC_LOONGSON) += dwmac-loongson.o stmmac-pci-objs:= stmmac_pci.o diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c new file mode 100644 index 0000000000000000000000000000000000000000..f3f3682eae78a06a27a051039e8b30282c9f407e --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Loongson Corporation + */ + +#include +#include +#include +#include +#include +#include "stmmac.h" + +struct stmmac_pci_info { + int (*setup)(struct pci_dev *pdev, struct plat_stmmacenet_data *plat); +}; + +static void common_default_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn); + plat->interface = PHY_INTERFACE_MODE_GMII; + + plat->clk_csr = 2; /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ + plat->has_gmac = 1; + plat->force_sf_dma_mode = 1; + + /* Set default value for multicast hash bins */ + plat->multicast_filter_bins = 256; + + /* Set default value for unicast filter entries */ + plat->unicast_filter_entries = 1; + + /* Set the maxmtu to a default of JUMBO_LEN */ + plat->maxmtu = JUMBO_LEN; + + /* Set default number of RX and TX queues to use */ + plat->tx_queues_to_use = 1; + plat->rx_queues_to_use = 1; + + /* Disable Priority config by default */ + plat->tx_queues_cfg[0].use_prio = false; + plat->rx_queues_cfg[0].use_prio = false; + + /* Disable RX queues routing by default */ + plat->rx_queues_cfg[0].pkt_route = 0x0; + + plat->dma_cfg->pbl = 32; + plat->dma_cfg->pblx8 = true; + + plat->clk_ref_rate = 125000000; + plat->clk_ptp_rate = 125000000; +} + +static int loongson_gmac_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + common_default_data(pdev, plat); + + plat->mdio_bus_data->phy_mask = 0; + + plat->phy_addr = -1; + plat->phy_interface = PHY_INTERFACE_MODE_RGMII_ID; + + return 0; +} + +static struct stmmac_pci_info loongson_gmac_pci_info = { + .setup = loongson_gmac_data, +}; + +static void loongson_gnet_fix_speed(void *priv, unsigned int speed) +{ + struct net_device *ndev = (struct net_device *)(*(unsigned long *)priv); + struct stmmac_priv *ptr = netdev_priv(ndev); + + if (speed == SPEED_1000) { + if (readl(ptr->ioaddr + MAC_CTRL_REG) & (1 << 15) /* PS */) { + /* reset phy */ + phy_set_bits(ndev->phydev, 0 /*MII_BMCR*/, + 0x200 /*BMCR_ANRESTART*/); + } + } +} + +static int loongson_gnet_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + common_default_data(pdev, plat); + + plat->mdio_bus_data->phy_mask = 0xfffffffb; + + plat->phy_addr = 2; + plat->phy_interface = PHY_INTERFACE_MODE_GMII; + + /* GNET 1000M speed need workaround */ + plat->fix_mac_speed = loongson_gnet_fix_speed; + + /* Get netdev pointer address */ + plat->bsp_priv = &(pdev->dev.driver_data); + + return 0; +} + +static struct stmmac_pci_info loongson_gnet_pci_info = { + .setup = loongson_gnet_data, +}; + +static int loongson_dwmac_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct plat_stmmacenet_data *plat; + struct stmmac_pci_info *info; + struct stmmac_resources res; + struct device_node *np; + int ret, i, bus_id, phy_mode; + bool mdio = false; + + np = dev_of_node(&pdev->dev); + if (np && !of_device_is_compatible(np, "loongson, pci-gmac")) { + pr_info("dwmac_loongson_pci: Incompatible OF node\n"); + return -ENODEV; + } + + plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); + if (!plat) + return -ENOMEM; + + if (plat->mdio_node) { + dev_err(&pdev->dev, "Found MDIO subnode\n"); + mdio = true; + } + + plat->mdio_bus_data = devm_kzalloc(&pdev->dev, + sizeof(*plat->mdio_bus_data), + GFP_KERNEL); + if (!plat->mdio_bus_data) + return -ENOMEM; + + if (mdio) + plat->mdio_bus_data->needs_reset = true; + + plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL); + if (!plat->dma_cfg) + return -ENOMEM; + + /* Enable pci device */ + ret = pci_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); + return ret; + } + + /* Get the base address of device */ + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret) + return ret; + break; + } + + pci_set_master(pdev); + + info = (struct stmmac_pci_info *)id->driver_data; + ret = info->setup(pdev, plat); + if (ret) + return ret; + + if (np) { + bus_id = of_alias_get_id(np, "ethernet"); + if (bus_id >= 0) + plat->bus_id = bus_id; + + phy_mode = device_get_phy_mode(&pdev->dev); + if (phy_mode < 0) { + dev_err(&pdev->dev, "phy_mode not found\n"); + return phy_mode; + } + plat->phy_interface = phy_mode; + } + + pci_enable_msi(pdev); + + memset(&res, 0, sizeof(res)); + res.addr = pcim_iomap_table(pdev)[0]; + if (np) { + res.irq = of_irq_get_byname(np, "macirq"); + if (res.irq < 0) { + dev_err(&pdev->dev, "IRQ macirq not found\n"); + ret = -ENODEV; + } + + res.wol_irq = of_irq_get_byname(np, "eth_wake_irq"); + if (res.wol_irq < 0) { + dev_info(&pdev->dev, + "IRQ eth_wake_irq not found, using macirq\n"); + res.wol_irq = res.irq; + } + + res.lpi_irq = of_irq_get_byname(np, "eth_lpi"); + if (res.lpi_irq < 0) { + dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); + ret = -ENODEV; + } + } else { + res.irq = pdev->irq; + res.wol_irq = pdev->irq; + } + + return stmmac_dvr_probe(&pdev->dev, plat, &res); +} + +static void loongson_dwmac_remove(struct pci_dev *pdev) +{ + int i; + + stmmac_dvr_remove(&pdev->dev); + + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + pcim_iounmap_regions(pdev, BIT(i)); + break; + } + + pci_disable_device(pdev); +} + +static int __maybe_unused loongson_dwmac_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = stmmac_suspend(dev); + if (ret) + return ret; + + ret = pci_save_state(pdev); + if (ret) + return ret; + + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + return 0; +} + +static int __maybe_unused loongson_dwmac_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pci_restore_state(pdev); + pci_set_power_state(pdev, PCI_D0); + + ret = pci_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + return stmmac_resume(dev); +} + +static SIMPLE_DEV_PM_OPS(loongson_dwmac_pm_ops, loongson_dwmac_suspend, + loongson_dwmac_resume); + +#define PCI_DEVICE_ID_LOONGSON_GMAC 0x7a03 +#define PCI_DEVICE_ID_LOONGSON_GNET 0x7a13 + +static const struct pci_device_id loongson_dwmac_id_table[] = { + { PCI_DEVICE_DATA(LOONGSON, GMAC, &loongson_gmac_pci_info) }, + { PCI_DEVICE_DATA(LOONGSON, GNET, &loongson_gnet_pci_info) }, + {} +}; +MODULE_DEVICE_TABLE(pci, loongson_dwmac_id_table); + +struct pci_driver loongson_dwmac_driver = { + .name = "dwmac-loongson-pci", + .id_table = loongson_dwmac_id_table, + .probe = loongson_dwmac_probe, + .remove = loongson_dwmac_remove, + .driver = { + .pm = &loongson_dwmac_pm_ops, + }, +}; + +module_pci_driver(loongson_dwmac_driver); + +MODULE_DESCRIPTION("Loongson DWMAC PCI driver"); +MODULE_AUTHOR("Qing Zhang "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 650c7d4be82ff9ea7c3bb1415195d39d086341eb..84b6c61b59aff5fdd4014f479297f7402769f12f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -74,7 +74,7 @@ MODULE_PARM_DESC(phyaddr, "Physical device address"); #define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4) #define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4) -static int flow_ctrl = FLOW_AUTO; +static int flow_ctrl = FLOW_OFF; module_param(flow_ctrl, int, 0644); MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); @@ -5182,7 +5182,8 @@ int stmmac_dvr_probe(struct device *device, pm_runtime_get_noresume(device); pm_runtime_set_active(device); - pm_runtime_enable(device); + if (!pm_runtime_enabled(device)) + pm_runtime_enable(device); if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) { diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index d10324431354275a7e7d1bddb54ee8571599d8e2..94e36deefe88acc76b7c46c6afee0b1ffdc85497 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -51,7 +51,7 @@ #define AM65_CPSW_MAX_PORTS 8 #define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN -#define AM65_CPSW_MAX_PACKET_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) +#define AM65_CPSW_MAX_PACKET_SIZE 2024 #define AM65_CPSW_REG_CTL 0x004 #define AM65_CPSW_REG_STAT_PORT_EN 0x014 @@ -1853,7 +1853,8 @@ static int am65_cpsw_nuss_init_ndev_2g(struct am65_cpsw_common *common) ether_addr_copy(port->ndev->dev_addr, port->slave.mac_addr); port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE; - port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE; + port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE - + (VLAN_ETH_HLEN + ETH_FCS_LEN); port->ndev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | NETIF_F_HW_CSUM | diff --git a/drivers/net/fjes/fjes_hw.c b/drivers/net/fjes/fjes_hw.c index 065bb0a40b1d1e3343bfaeadc3eb85b8a129f551..a1405a3e294c34cff27af683dd0656972dde2f51 100644 --- a/drivers/net/fjes/fjes_hw.c +++ b/drivers/net/fjes/fjes_hw.c @@ -220,21 +220,25 @@ static int fjes_hw_setup(struct fjes_hw *hw) mem_size = FJES_DEV_REQ_BUF_SIZE(hw->max_epid); hw->hw_info.req_buf = kzalloc(mem_size, GFP_KERNEL); - if (!(hw->hw_info.req_buf)) - return -ENOMEM; + if (!(hw->hw_info.req_buf)) { + result = -ENOMEM; + goto free_ep_info; + } hw->hw_info.req_buf_size = mem_size; mem_size = FJES_DEV_RES_BUF_SIZE(hw->max_epid); hw->hw_info.res_buf = kzalloc(mem_size, GFP_KERNEL); - if (!(hw->hw_info.res_buf)) - return -ENOMEM; + if (!(hw->hw_info.res_buf)) { + result = -ENOMEM; + goto free_req_buf; + } hw->hw_info.res_buf_size = mem_size; result = fjes_hw_alloc_shared_status_region(hw); if (result) - return result; + goto free_res_buf; hw->hw_info.buffer_share_bit = 0; hw->hw_info.buffer_unshare_reserve_bit = 0; @@ -245,11 +249,11 @@ static int fjes_hw_setup(struct fjes_hw *hw) result = fjes_hw_alloc_epbuf(&buf_pair->tx); if (result) - return result; + goto free_epbuf; result = fjes_hw_alloc_epbuf(&buf_pair->rx); if (result) - return result; + goto free_epbuf; spin_lock_irqsave(&hw->rx_status_lock, flags); fjes_hw_setup_epbuf(&buf_pair->tx, mac, @@ -272,6 +276,25 @@ static int fjes_hw_setup(struct fjes_hw *hw) fjes_hw_init_command_registers(hw, ¶m); return 0; + +free_epbuf: + for (epidx = 0; epidx < hw->max_epid ; epidx++) { + if (epidx == hw->my_epid) + continue; + fjes_hw_free_epbuf(&hw->ep_shm_info[epidx].tx); + fjes_hw_free_epbuf(&hw->ep_shm_info[epidx].rx); + } + fjes_hw_free_shared_status_region(hw); +free_res_buf: + kfree(hw->hw_info.res_buf); + hw->hw_info.res_buf = NULL; +free_req_buf: + kfree(hw->hw_info.req_buf); + hw->hw_info.req_buf = NULL; +free_ep_info: + kfree(hw->ep_shm_info); + hw->ep_shm_info = NULL; + return result; } static void fjes_hw_cleanup(struct fjes_hw *hw) diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index bbbe198f83e88aae1b3a25dc500b28259a89ccba..2b7616f161d693b234dcaf361797476a7d2d249c 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -1345,6 +1345,7 @@ static struct phy_driver ksphy_driver[] = { /* PHY_GBIT_FEATURES */ .driver_data = &ksz9021_type, .probe = kszphy_probe, + .soft_reset = genphy_soft_reset, .config_init = ksz9131_config_init, .read_status = genphy_read_status, .ack_interrupt = kszphy_ack_interrupt, diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c index 622fc7f170402c63a57def23845d6a2daafaa943..5037142c5a822e4a12c6d459b0bf7c3ad9406cfc 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c @@ -647,9 +647,10 @@ void ath9k_htc_txstatus(struct ath9k_htc_priv *priv, void *wmi_event) struct ath9k_htc_tx_event *tx_pend; int i; - for (i = 0; i < txs->cnt; i++) { - WARN_ON(txs->cnt > HTC_MAX_TX_STATUS); + if (WARN_ON_ONCE(txs->cnt > HTC_MAX_TX_STATUS)) + return; + for (i = 0; i < txs->cnt; i++) { __txs = &txs->txstatus[i]; skb = ath9k_htc_tx_get_packet(priv, __txs); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index fdf2c6ea41d96e70144a32968e0077c2a72f4658..bcaec8a184cd674c883af16481f7918faa34de48 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -876,7 +876,7 @@ static int iwl_dbg_tlv_override_trig_node(struct iwl_fw_runtime *fwrt, node_trig = (void *)node_tlv->data; } - memcpy(node_trig->data + offset, trig->data, trig_data_len); + memcpy((u8 *)node_trig->data + offset, trig->data, trig_data_len); node_tlv->length = cpu_to_le32(size); if (policy & IWL_FW_INI_APPLY_POLICY_OVERRIDE_CFG) { diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index 3d1b5d3d295aedef24ea817ff3a6bf26eea380fc..2f5f1ff22a601dd11d53868080c4d067c12a419b 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -1980,6 +1980,8 @@ static int mwifiex_cfg80211_start_ap(struct wiphy *wiphy, mwifiex_set_sys_config_invalid_data(bss_cfg); + memcpy(bss_cfg->mac_addr, priv->curr_addr, ETH_ALEN); + if (params->beacon_interval) bss_cfg->beacon_period = params->beacon_interval; if (params->dtim_period) diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h index 470d669c7f149b117465dad9939feba9a2bf2be3..96c42b979e9beece67cd550a2d847995d7ec8c35 100644 --- a/drivers/net/wireless/marvell/mwifiex/fw.h +++ b/drivers/net/wireless/marvell/mwifiex/fw.h @@ -177,6 +177,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER { #define TLV_TYPE_STA_MAC_ADDR (PROPRIETARY_TLV_BASE_ID + 32) #define TLV_TYPE_BSSID (PROPRIETARY_TLV_BASE_ID + 35) #define TLV_TYPE_CHANNELBANDLIST (PROPRIETARY_TLV_BASE_ID + 42) +#define TLV_TYPE_UAP_MAC_ADDRESS (PROPRIETARY_TLV_BASE_ID + 43) #define TLV_TYPE_UAP_BEACON_PERIOD (PROPRIETARY_TLV_BASE_ID + 44) #define TLV_TYPE_UAP_DTIM_PERIOD (PROPRIETARY_TLV_BASE_ID + 45) #define TLV_TYPE_UAP_BCAST_SSID (PROPRIETARY_TLV_BASE_ID + 48) diff --git a/drivers/net/wireless/marvell/mwifiex/ioctl.h b/drivers/net/wireless/marvell/mwifiex/ioctl.h index 3db449efa167cbbd61026eebe2e88fb73fa2a5ba..cdb5b3881782ff2a00c28e582c3a9865efebd851 100644 --- a/drivers/net/wireless/marvell/mwifiex/ioctl.h +++ b/drivers/net/wireless/marvell/mwifiex/ioctl.h @@ -119,6 +119,7 @@ struct mwifiex_uap_bss_param { u8 qos_info; u8 power_constraint; struct mwifiex_types_wmm_info wmm_info; + u8 mac_addr[ETH_ALEN]; }; enum { diff --git a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c index b48a85d791f687510ea6b89c7a432a92f1ea2180..2d444e660ac5993aa1e2e6957796f518c159c150 100644 --- a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c +++ b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c @@ -479,6 +479,7 @@ void mwifiex_config_uap_11d(struct mwifiex_private *priv, static int mwifiex_uap_bss_param_prepare(u8 *tlv, void *cmd_buf, u16 *param_size) { + struct host_cmd_tlv_mac_addr *mac_tlv; struct host_cmd_tlv_dtim_period *dtim_period; struct host_cmd_tlv_beacon_period *beacon_period; struct host_cmd_tlv_ssid *ssid; @@ -498,6 +499,13 @@ mwifiex_uap_bss_param_prepare(u8 *tlv, void *cmd_buf, u16 *param_size) int i; u16 cmd_size = *param_size; + mac_tlv = (struct host_cmd_tlv_mac_addr *)tlv; + mac_tlv->header.type = cpu_to_le16(TLV_TYPE_UAP_MAC_ADDRESS); + mac_tlv->header.len = cpu_to_le16(ETH_ALEN); + memcpy(mac_tlv->mac_addr, bss_cfg->mac_addr, ETH_ALEN); + cmd_size += sizeof(struct host_cmd_tlv_mac_addr); + tlv += sizeof(struct host_cmd_tlv_mac_addr); + if (bss_cfg->ssid.ssid_len) { ssid = (struct host_cmd_tlv_ssid *)tlv; ssid->header.type = cpu_to_le16(TLV_TYPE_UAP_SSID); diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c index b04f76551ca480777adcc45035d8e3ec54af9fd1..be3c153ab3b0bf131dd3b2d023555fefb78330e6 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c @@ -101,6 +101,7 @@ void rt2x00lib_disable_radio(struct rt2x00_dev *rt2x00dev) rt2x00link_stop_tuner(rt2x00dev); rt2x00queue_stop_queues(rt2x00dev); rt2x00queue_flush_queues(rt2x00dev, true); + rt2x00queue_stop_queue(rt2x00dev->bcn); /* * Disable radio. @@ -1272,6 +1273,7 @@ int rt2x00lib_start(struct rt2x00_dev *rt2x00dev) rt2x00dev->intf_ap_count = 0; rt2x00dev->intf_sta_count = 0; rt2x00dev->intf_associated = 0; + rt2x00dev->intf_beaconing = 0; /* Enable the radio */ retval = rt2x00lib_enable_radio(rt2x00dev); @@ -1298,6 +1300,7 @@ void rt2x00lib_stop(struct rt2x00_dev *rt2x00dev) rt2x00dev->intf_ap_count = 0; rt2x00dev->intf_sta_count = 0; rt2x00dev->intf_associated = 0; + rt2x00dev->intf_beaconing = 0; } static inline void rt2x00lib_set_if_combinations(struct rt2x00_dev *rt2x00dev) diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c index 2f68a31072ae4f61008c8c039361184b962c2027..795bd3b0ebd8f9f929063e354d94f06eed8bef8c 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c @@ -599,6 +599,17 @@ void rt2x00mac_bss_info_changed(struct ieee80211_hw *hw, */ if (changes & BSS_CHANGED_BEACON_ENABLED) { mutex_lock(&intf->beacon_skb_mutex); + + /* + * Clear the 'enable_beacon' flag and clear beacon because + * the beacon queue has been stopped after hardware reset. + */ + if (test_bit(DEVICE_STATE_RESET, &rt2x00dev->flags) && + intf->enable_beacon) { + intf->enable_beacon = false; + rt2x00queue_clear_beacon(rt2x00dev, vif); + } + if (!bss_conf->enable_beacon && intf->enable_beacon) { rt2x00dev->intf_beaconing--; intf->enable_beacon = false; diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index 3776495fd9d03f024abb5a6487af1147397358d9..679ae786cf4507cd7c43b2e3c5935d67ae034e77 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -164,21 +164,29 @@ static bool _rtl_pci_platform_switch_device_pci_aspm( struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + value &= PCI_EXP_LNKCTL_ASPMC; + if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192SE) - value |= 0x40; + value |= PCI_EXP_LNKCTL_CCC; - pci_write_config_byte(rtlpci->pdev, 0x80, value); + pcie_capability_clear_and_set_word(rtlpci->pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC | value, + value); return false; } -/*When we set 0x01 to enable clk request. Set 0x0 to disable clk req.*/ -static void _rtl_pci_switch_clk_req(struct ieee80211_hw *hw, u8 value) +/* @value is PCI_EXP_LNKCTL_CLKREQ_EN or 0 to enable/disable clk request. */ +static void _rtl_pci_switch_clk_req(struct ieee80211_hw *hw, u16 value) { struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); - pci_write_config_byte(rtlpci->pdev, 0x81, value); + value &= PCI_EXP_LNKCTL_CLKREQ_EN; + + pcie_capability_clear_and_set_word(rtlpci->pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN, + value); if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) udelay(100); @@ -192,11 +200,8 @@ static void rtl_pci_disable_aspm(struct ieee80211_hw *hw) struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor; - u8 num4bytes = pcipriv->ndis_adapter.num4bytes; /*Retrieve original configuration settings. */ u8 linkctrl_reg = pcipriv->ndis_adapter.linkctrl_reg; - u16 pcibridge_linkctrlreg = pcipriv->ndis_adapter. - pcibridge_linkctrlreg; u16 aspmlevel = 0; u8 tmp_u1b = 0; @@ -221,16 +226,8 @@ static void rtl_pci_disable_aspm(struct ieee80211_hw *hw) /*Set corresponding value. */ aspmlevel |= BIT(0) | BIT(1); linkctrl_reg &= ~aspmlevel; - pcibridge_linkctrlreg &= ~(BIT(0) | BIT(1)); _rtl_pci_platform_switch_device_pci_aspm(hw, linkctrl_reg); - udelay(50); - - /*4 Disable Pci Bridge ASPM */ - pci_write_config_byte(rtlpci->pdev, (num4bytes << 2), - pcibridge_linkctrlreg); - - udelay(50); } /*Enable RTL8192SE ASPM & Enable Pci Bridge ASPM for @@ -245,9 +242,7 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor; - u8 num4bytes = pcipriv->ndis_adapter.num4bytes; u16 aspmlevel; - u8 u_pcibridge_aspmsetting; u8 u_device_aspmsetting; if (!ppsc->support_aspm) @@ -259,25 +254,6 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) return; } - /*4 Enable Pci Bridge ASPM */ - - u_pcibridge_aspmsetting = - pcipriv->ndis_adapter.pcibridge_linkctrlreg | - rtlpci->const_hostpci_aspm_setting; - - if (pcibridge_vendor == PCI_BRIDGE_VENDOR_INTEL) - u_pcibridge_aspmsetting &= ~BIT(0); - - pci_write_config_byte(rtlpci->pdev, (num4bytes << 2), - u_pcibridge_aspmsetting); - - rtl_dbg(rtlpriv, COMP_INIT, DBG_LOUD, - "PlatformEnableASPM(): Write reg[%x] = %x\n", - (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10), - u_pcibridge_aspmsetting); - - udelay(50); - /*Get ASPM level (with/without Clock Req) */ aspmlevel = rtlpci->const_devicepci_aspm_setting; u_device_aspmsetting = pcipriv->ndis_adapter.linkctrl_reg; @@ -291,7 +267,8 @@ static void rtl_pci_enable_aspm(struct ieee80211_hw *hw) if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_CLK_REQ) { _rtl_pci_switch_clk_req(hw, (ppsc->reg_rfps_level & - RT_RF_OFF_LEVL_CLK_REQ) ? 1 : 0); + RT_RF_OFF_LEVL_CLK_REQ) ? + PCI_EXP_LNKCTL_CLKREQ_EN : 0); RT_SET_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_CLK_REQ); } udelay(100); @@ -359,22 +336,6 @@ static bool rtl_pci_check_buddy_priv(struct ieee80211_hw *hw, return find_buddy_priv; } -static void rtl_pci_get_linkcontrol_field(struct ieee80211_hw *hw) -{ - struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw); - struct rtl_pci *rtlpci = rtl_pcidev(pcipriv); - u8 capabilityoffset = pcipriv->ndis_adapter.pcibridge_pciehdr_offset; - u8 linkctrl_reg; - u8 num4bbytes; - - num4bbytes = (capabilityoffset + 0x10) / 4; - - /*Read Link Control Register */ - pci_read_config_byte(rtlpci->pdev, (num4bbytes << 2), &linkctrl_reg); - - pcipriv->ndis_adapter.pcibridge_linkctrlreg = linkctrl_reg; -} - static void rtl_pci_parse_configuration(struct pci_dev *pdev, struct ieee80211_hw *hw) { @@ -2035,12 +1996,6 @@ static bool _rtl_pci_find_adapter(struct pci_dev *pdev, PCI_SLOT(bridge_pdev->devfn); pcipriv->ndis_adapter.pcibridge_funcnum = PCI_FUNC(bridge_pdev->devfn); - pcipriv->ndis_adapter.pcibridge_pciehdr_offset = - pci_pcie_cap(bridge_pdev); - pcipriv->ndis_adapter.num4bytes = - (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10) / 4; - - rtl_pci_get_linkcontrol_field(hw); if (pcipriv->ndis_adapter.pcibridge_vendor == PCI_BRIDGE_VENDOR_AMD) { @@ -2057,13 +2012,11 @@ static bool _rtl_pci_find_adapter(struct pci_dev *pdev, pdev->vendor, pcipriv->ndis_adapter.linkctrl_reg); rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, - "pci_bridge busnumber:devnumber:funcnumber:vendor:pcie_cap:link_ctl_reg:amd %d:%d:%d:%x:%x:%x:%x\n", + "pci_bridge busnumber:devnumber:funcnumber:vendor:amd %d:%d:%d:%x:%x\n", pcipriv->ndis_adapter.pcibridge_busnum, pcipriv->ndis_adapter.pcibridge_devnum, pcipriv->ndis_adapter.pcibridge_funcnum, pcibridge_vendors[pcipriv->ndis_adapter.pcibridge_vendor], - pcipriv->ndis_adapter.pcibridge_pciehdr_offset, - pcipriv->ndis_adapter.pcibridge_linkctrlreg, pcipriv->ndis_adapter.amd_l1_patch); rtl_pci_parse_configuration(pdev, hw); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.h b/drivers/net/wireless/realtek/rtlwifi/pci.h index 866861626a0a1b2d58f7b6ba44fb9a6eb187f8d5..d6307197dfea06bc0b35f25560d30b592f4e6cca 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.h +++ b/drivers/net/wireless/realtek/rtlwifi/pci.h @@ -236,11 +236,6 @@ struct mp_adapter { u16 pcibridge_vendorid; u16 pcibridge_deviceid; - u8 num4bytes; - - u8 pcibridge_pciehdr_offset; - u8 pcibridge_linkctrlreg; - bool amd_l1_patch; }; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 153150e8beb9493807f838005163624d19c6385d..116ae6fd35e2d7264a601a0c0b124406074fb33a 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -919,7 +919,7 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) { struct nvme_tcp_data_pdu *data = &queue->pdu.data; struct nvmet_tcp_cmd *cmd; - unsigned int plen; + unsigned int exp_data_len; if (likely(queue->nr_cmds)) { if (unlikely(data->ttag >= queue->nr_cmds)) { @@ -938,14 +938,17 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) data->ttag, le32_to_cpu(data->data_offset), cmd->rbytes_done); /* FIXME: use path and transport errors */ - nvmet_req_complete(&cmd->req, - NVME_SC_INVALID_FIELD | NVME_SC_DNR); + nvmet_tcp_fatal_error(queue); return -EPROTO; } - plen = le32_to_cpu(data->hdr.plen); + exp_data_len = le32_to_cpu(data->hdr.plen) - + nvmet_tcp_hdgst_len(queue) - + nvmet_tcp_ddgst_len(queue) - + sizeof(*data); + cmd->pdu_len = le32_to_cpu(data->data_length); - if (unlikely(cmd->pdu_len != (plen - sizeof(*data)) || + if (unlikely(cmd->pdu_len != exp_data_len || cmd->pdu_len == 0 || cmd->pdu_len > NVMET_TCP_MAXH2CDATA)) { pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len); diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 197abe33b65ce7dd08581446baed7555ed80c919..f9083c868a36d68e5c5960e36eaee19782de73ca 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -565,7 +565,7 @@ static void __init of_unittest_parse_phandle_with_args_map(void) } rc = of_count_phandle_with_args(np, "phandle-list", "#phandle-cells"); - unittest(rc == 8, "of_count_phandle_with_args() returned %i, expected 7\n", rc); + unittest(rc == 8, "of_count_phandle_with_args() returned %i, expected 8\n", rc); for (i = 0; i < 9; i++) { bool passed = true; diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa390229e2557ac15d09dc6b4fc402..5c4e184ac9dc88371338e965db13b017f7b80f42 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -291,7 +291,7 @@ config PCI_HYPERV_INTERFACE config PCI_LOONGSON bool "LOONGSON PCI Controller" depends on MACH_LOONGSON64 || COMPILE_TEST - depends on OF + depends on OF || ACPI depends on PCI_QUIRKS default MACH_LOONGSON64 help diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index afaea201a5afce3702f0669a5ae4817f47dc7eeb..d3c3ca3ef4bae0dcd283bdea7eb5291f964b6aa2 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -1261,7 +1261,16 @@ static int ks_pcie_probe(struct platform_device *pdev) goto err_link; } + /* Obtain references to the PHYs */ + for (i = 0; i < num_lanes; i++) + phy_pm_runtime_get_sync(ks_pcie->phy[i]); + ret = ks_pcie_enable_phy(ks_pcie); + + /* Release references to the PHYs */ + for (i = 0; i < num_lanes; i++) + phy_pm_runtime_put_sync(ks_pcie->phy[i]); + if (ret) { dev_err(dev, "failed to enable phy\n"); goto err_link; diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c index 8b40b45590a0214f826c4951bd8c5d2536004b7f..a3af81f54aadefd1511939d73f539fa03bb47c9f 100644 --- a/drivers/pci/controller/pci-loongson.c +++ b/drivers/pci/controller/pci-loongson.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "../pci.h" @@ -30,11 +32,16 @@ #define FLAG_CFG1 BIT(1) #define FLAG_DEV_FIX BIT(2) +struct pci_controller_data { + u32 flags; + struct pci_ops *ops; +}; + struct loongson_pci { void __iomem *cfg0_base; void __iomem *cfg1_base; struct platform_device *pdev; - u32 flags; + struct pci_controller_data *data; }; /* Fixup wrong class code in PCIe bridges */ @@ -129,55 +136,101 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, DEV_LS7A_PCIE_PORT6, loongson_mrrs_quirk); -static void __iomem *cfg1_map(struct loongson_pci *priv, int bus, - unsigned int devfn, int where) +static void loongson_pci_pin_quirk(struct pci_dev *pdev) { - unsigned long addroff = 0x0; + pdev->pin = 1 + (PCI_FUNC(pdev->devfn) & 3); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_AHCI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_EHCI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_OHCI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_DC1, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_DC2, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_GPU, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_HDMI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_GMAC, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + PCI_DEVICE_ID_LOONGSON_GNET, loongson_pci_pin_quirk); + +static struct loongson_pci *pci_bus_to_loongson_pci(struct pci_bus *bus) +{ + struct pci_config_window *cfg; - if (bus != 0) - addroff |= BIT(28); /* Type 1 Access */ - addroff |= (where & 0xff) | ((where & 0xf00) << 16); - addroff |= (bus << 16) | (devfn << 8); - return priv->cfg1_base + addroff; + if (acpi_disabled) + return (struct loongson_pci *)(bus->sysdata); + + cfg = bus->sysdata; + return (struct loongson_pci *)(cfg->priv); } -static void __iomem *cfg0_map(struct loongson_pci *priv, int bus, - unsigned int devfn, int where) +static void __iomem *cfg0_map(struct loongson_pci *priv, + struct pci_bus *bus, unsigned int devfn, int where) { unsigned long addroff = 0x0; + unsigned char busnum = bus->number; - if (bus != 0) + if (!pci_is_root_bus(bus)) { addroff |= BIT(24); /* Type 1 Access */ - addroff |= (bus << 16) | (devfn << 8) | where; + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | where; return priv->cfg0_base + addroff; } +static void __iomem *cfg1_map(struct loongson_pci *priv, + struct pci_bus *bus, unsigned int devfn, int where) +{ + unsigned long addroff = 0x0; + unsigned char busnum = bus->number; + + if (!pci_is_root_bus(bus)) { + addroff |= BIT(28); /* Type 1 Access */ + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | (where & 0xff) | ((where & 0xf00) << 16); + return priv->cfg1_base + addroff; +} + static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { - unsigned char busnum = bus->number; - struct pci_host_bridge *bridge = pci_find_host_bridge(bus); - struct loongson_pci *priv = pci_host_bridge_priv(bridge); + unsigned int device = PCI_SLOT(devfn); + unsigned int function = PCI_FUNC(devfn); + struct loongson_pci *priv = pci_bus_to_loongson_pci(bus); /* * Do not read more than one device on the bus other than * the host bus. For our hardware the root bus is always bus 0. */ - if (priv->flags & FLAG_DEV_FIX && busnum != 0 && - PCI_SLOT(devfn) > 0) + if ((priv->data->flags & FLAG_DEV_FIX) && bus->self) { + if (!pci_is_root_bus(bus) && (device > 0)) + return NULL; + } + + /* Don't access unexisting devices */ + if (pci_is_root_bus(bus) && (device >= 9 && device <= 20 && function > 0)) return NULL; /* CFG0 can only access standard space */ if (where < PCI_CFG_SPACE_SIZE && priv->cfg0_base) - return cfg0_map(priv, busnum, devfn, where); + return cfg0_map(priv, bus, devfn, where); /* CFG1 can access extended space */ if (where < PCI_CFG_SPACE_EXP_SIZE && priv->cfg1_base) - return cfg1_map(priv, busnum, devfn, where); + return cfg1_map(priv, bus, devfn, where); return NULL; } +#ifdef CONFIG_OF + static int loongson_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { int irq; @@ -196,20 +249,42 @@ static int loongson_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return val; } -/* H/w only accept 32-bit PCI operations */ +/* LS2K/LS7A accept 8/16/32-bit PCI config operations */ static struct pci_ops loongson_pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +/* RS780/SR5690 only accept 32-bit PCI config operations */ +static struct pci_ops loongson_pci_ops32 = { .map_bus = pci_loongson_map_bus, .read = pci_generic_config_read32, .write = pci_generic_config_write32, }; +static const struct pci_controller_data ls2k_pci_data = { + .flags = FLAG_CFG1 | FLAG_DEV_FIX, + .ops = &loongson_pci_ops, +}; + +static const struct pci_controller_data ls7a_pci_data = { + .flags = FLAG_CFG1 | FLAG_DEV_FIX, + .ops = &loongson_pci_ops, +}; + +static const struct pci_controller_data rs780e_pci_data = { + .flags = FLAG_CFG0, + .ops = &loongson_pci_ops32, +}; + static const struct of_device_id loongson_pci_of_match[] = { { .compatible = "loongson,ls2k-pci", - .data = (void *)(FLAG_CFG0 | FLAG_CFG1 | FLAG_DEV_FIX), }, + .data = (void *)&ls2k_pci_data, }, { .compatible = "loongson,ls7a-pci", - .data = (void *)(FLAG_CFG0 | FLAG_CFG1 | FLAG_DEV_FIX), }, + .data = (void *)&ls7a_pci_data, }, { .compatible = "loongson,rs780e-pci", - .data = (void *)(FLAG_CFG0), }, + .data = (void *)&rs780e_pci_data, }, {} }; @@ -230,20 +305,20 @@ static int loongson_pci_probe(struct platform_device *pdev) priv = pci_host_bridge_priv(bridge); priv->pdev = pdev; - priv->flags = (unsigned long)of_device_get_match_data(dev); + priv->data = (struct pci_controller_data *)of_device_get_match_data(dev); - regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!regs) { - dev_err(dev, "missing mem resources for cfg0\n"); - return -EINVAL; + if (priv->data->flags & FLAG_CFG0) { + regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs) + dev_err(dev, "missing mem resources for cfg0\n"); + else { + priv->cfg0_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(priv->cfg0_base)) + return PTR_ERR(priv->cfg0_base); + } } - priv->cfg0_base = devm_pci_remap_cfg_resource(dev, regs); - if (IS_ERR(priv->cfg0_base)) - return PTR_ERR(priv->cfg0_base); - - /* CFG1 is optional */ - if (priv->flags & FLAG_CFG1) { + if (priv->data->flags & FLAG_CFG1) { regs = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (!regs) dev_info(dev, "missing mem resource for cfg1\n"); @@ -255,7 +330,7 @@ static int loongson_pci_probe(struct platform_device *pdev) } bridge->sysdata = priv; - bridge->ops = &loongson_pci_ops; + bridge->ops = priv->data->ops; bridge->map_irq = loongson_map_irq; return pci_host_probe(bridge); @@ -269,3 +344,41 @@ static struct platform_driver loongson_pci_driver = { .probe = loongson_pci_probe, }; builtin_platform_driver(loongson_pci_driver); + +#endif + +#ifdef CONFIG_ACPI + +static int loongson_pci_ecam_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct loongson_pci *priv; + struct pci_controller_data *data; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + cfg->priv = priv; + data->flags = FLAG_CFG1; + priv->data = data; + priv->cfg1_base = cfg->win - (cfg->busr.start << 16); + + return 0; +} + +const struct pci_ecam_ops loongson_pci_ecam_ops = { + .bus_shift = 16, + .init = loongson_pci_ecam_init, + .pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +#endif diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 23548b517e4b600b79c511008ed2e12c70d74de7..ea91d63c8be157b45480266a768472590411b8dc 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -620,14 +620,20 @@ static void mtk_pcie_intr_handler(struct irq_desc *desc) if (status & MSI_STATUS){ unsigned long imsi_status; + /* + * The interrupt status can be cleared even if the + * MSI status remains pending. As such, given the + * edge-triggered interrupt type, its status should + * be cleared before being dispatched to the + * handler of the underlying device. + */ + writel(MSI_STATUS, port->base + PCIE_INT_STATUS); while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) { for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) { virq = irq_find_mapping(port->inner_domain, bit); generic_handle_irq(virq); } } - /* Clear MSI interrupt status */ - writel(MSI_STATUS, port->base + PCIE_INT_STATUS); } } diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index 2ff5724b8f13f06aed1d879d6286944679d9a568..358d7281f6e87fe3197563d7de23105188a234c9 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -117,7 +117,7 @@ int pcie_port_device_resume(struct device *dev); int pcie_port_device_runtime_suspend(struct device *dev); int pcie_port_device_runtime_resume(struct device *dev); #endif -void pcie_port_device_remove(struct pci_dev *dev); +void pcie_port_device_remove(struct pci_dev *dev, bool disable); int __must_check pcie_port_bus_register(void); void pcie_port_bus_unregister(void); diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c index 3779b264dbec39e322e54b96e4525120eb951334..682c1cce59659f59def8d520537912be663d9f86 100644 --- a/drivers/pci/pcie/portdrv_core.c +++ b/drivers/pci/pcie/portdrv_core.c @@ -492,11 +492,13 @@ EXPORT_SYMBOL_GPL(pcie_port_find_device); * Remove PCI Express port service devices associated with given port and * disable MSI-X or MSI for the port. */ -void pcie_port_device_remove(struct pci_dev *dev) +void pcie_port_device_remove(struct pci_dev *dev, bool disable) { device_for_each_child(&dev->dev, NULL, remove_iter); pci_free_irq_vectors(dev); - pci_disable_device(dev); + + if (disable) + pci_disable_device(dev); } /** diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index aac1a6828b4f9cefb9acb4faafa629dde2f4d4c6..f46411507f9cbc10811874971882232316ec4388 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -144,7 +144,18 @@ static void pcie_portdrv_remove(struct pci_dev *dev) pm_runtime_dont_use_autosuspend(&dev->dev); } - pcie_port_device_remove(dev); + pcie_port_device_remove(dev, true); +} + +static void pcie_portdrv_shutdown(struct pci_dev *dev) +{ + if (pci_bridge_d3_possible(dev)) { + pm_runtime_forbid(&dev->dev); + pm_runtime_get_noresume(&dev->dev); + pm_runtime_dont_use_autosuspend(&dev->dev); + } + + pcie_port_device_remove(dev, false); } static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, @@ -215,7 +226,7 @@ static struct pci_driver pcie_portdriver = { .probe = pcie_portdrv_probe, .remove = pcie_portdrv_remove, - .shutdown = pcie_portdrv_remove, + .shutdown = pcie_portdrv_shutdown, .err_handler = &pcie_portdrv_err_handler, diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 21661feeeeb653e0f4c3a8763157a642d214c749..2aa17ae72b74370005d5aeb97bf165fba4914163 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -29,6 +29,7 @@ #include #include #include +#include #include /* isa_dma_bridge_buggy */ #include "pci.h" @@ -206,6 +207,50 @@ static void quirk_mmio_always_on(struct pci_dev *dev) DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_BRIDGE_HOST, 8, quirk_mmio_always_on); +static void aspeed_fixup_vgaarb(struct pci_dev *pdev) +{ + struct pci_dev *bridge; + struct pci_bus *bus; + struct pci_dev *vdevp = NULL; + u16 config; + + bus = pdev->bus; + bridge = bus->self; + + /* Is VGA routed to us? */ + if (bridge && (pci_is_bridge(bridge))) { + pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, &config); + + /* Yes, this bridge is PCI bridge-to-bridge spec compliant, + * just return! + */ + if (config & PCI_BRIDGE_CTL_VGA) + return; + + dev_warn(&pdev->dev, "VGA bridge control is not enabled\n"); + } + + /* Just return if the system already have a default device */ + if (vga_default_device()) + return; + + /* No default vga device */ + while ((vdevp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, vdevp))) { + if (vdevp->vendor != 0x1a03) { + /* Have other vga devcie in the system, do nothing */ + dev_info(&pdev->dev, "Another boot vga device: 0x%x:0x%x\n", + vdevp->vendor, vdevp->device); + return; + } + } + + vga_set_default_device(pdev); + + dev_info(&pdev->dev, "Boot vga device set as 0x%x:0x%x\n", + pdev->vendor, pdev->device); +} +DECLARE_PCI_FIXUP_CLASS_FINAL(0x1a03, 0x2000, PCI_CLASS_DISPLAY_VGA, 8, aspeed_fixup_vgaarb); + /* * The Mellanox Tavor device gives false positive parity errors. Mark this * device with a broken_parity_status to allow PCI scanning code to "skip" @@ -3638,6 +3683,19 @@ static void quirk_no_pm_reset(struct pci_dev *dev) DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_ATI, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA, 8, quirk_no_pm_reset); +/* + * Spectrum-{1,2,3,4} devices report that a D3hot->D0 transition causes a reset + * (i.e., they advertise NoSoftRst-). However, this transition does not have + * any effect on the device: It continues to be operational and network ports + * remain up. Advertising this support makes it seem as if a PM reset is viable + * for these devices. Mark it as unavailable to skip it when testing reset + * methods. + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcb84, quirk_no_pm_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf6c, quirk_no_pm_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf70, quirk_no_pm_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf80, quirk_no_pm_reset); + /* * Thunderbolt controllers with broken MSI hotplug signaling: * Entire 1st generation (Light Ridge, Eagle Ridge, Light Peak) and part diff --git a/drivers/platform/Kconfig b/drivers/platform/Kconfig index 971426bb4302c9583d1765d92d6b924863a70453..5920903485aa4d828276ccda73c72c33da595722 100644 --- a/drivers/platform/Kconfig +++ b/drivers/platform/Kconfig @@ -6,6 +6,8 @@ if MIPS source "drivers/platform/mips/Kconfig" endif +source "drivers/platform/loongarch/Kconfig" + source "drivers/platform/goldfish/Kconfig" source "drivers/platform/chrome/Kconfig" diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index 6fda58c021ca4a3bced00822ce35da45636478b7..c36d65ac4c4dd899284f3bfcdcd2da861baa6375 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_X86) += x86/ +obj-$(CONFIG_LOONGARCH) += loongarch/ obj-$(CONFIG_MELLANOX_PLATFORM) += mellanox/ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_OLPC_EC) += olpc/ diff --git a/drivers/platform/loongarch/Kconfig b/drivers/platform/loongarch/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..3964c147c2e9bc52934825bd50061bebb3622015 --- /dev/null +++ b/drivers/platform/loongarch/Kconfig @@ -0,0 +1,38 @@ +# +# LoongArch Platform Specific Drivers +# + +menuconfig LOONGARCH_PLATFORM_DEVICES + bool "LoongArch Platform Specific Device Drivers" + default y + depends on LOONGARCH + help + Say Y here to get to see options for device drivers of various + LoongArch platforms, including vendor-specific laptop/desktop + extension and hardware monitor drivers. This option itself does + not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if LOONGARCH_PLATFORM_DEVICES + +config CPU_HWMON + bool "Loongson CPU HWMon Driver" + depends on MACH_LOONGSON64 + select HWMON + default y + help + Loongson-3A/3B/3C CPU Hwmon (temperature sensor) driver. + +config LOONGSON_LAPTOP + tristate "Generic Loongson-3A Laptop Driver" + depends on ACPI + depends on BACKLIGHT_CLASS_DEVICE + depends on INPUT + depends on MACH_LOONGSON64 + select INPUT_SPARSEKMAP + default y + help + ACPI-based Loongson-3 family laptops generic driver. + +endif # LOONGARCH_PLATFORM_DEVICES diff --git a/drivers/platform/loongarch/Makefile b/drivers/platform/loongarch/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8038291bae3aa82cb3f9b08c6c5bb9f364a6781c --- /dev/null +++ b/drivers/platform/loongarch/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CPU_HWMON) += cpu_hwmon.o +obj-$(CONFIG_LOONGSON_LAPTOP) += loongson-laptop.o diff --git a/drivers/platform/loongarch/cpu_hwmon.c b/drivers/platform/loongarch/cpu_hwmon.c new file mode 100644 index 0000000000000000000000000000000000000000..3b736a6c0ac7a3861f16635fafd59b104be5dfca --- /dev/null +++ b/drivers/platform/loongarch/cpu_hwmon.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Loongson Technology Co., Ltd. + */ +#include +#include +#include +#include +#include +#include + +#include + +static int nr_packages; +static struct device *cpu_hwmon_dev; + +static int loongson3_cpu_temp(int cpu) +{ + u32 reg; + + reg = iocsr_read32(LOONGARCH_IOCSR_CPUTEMP) & 0xff; + + return (int)((s8)reg) * 1000; +} + +static ssize_t cpu_temp_label(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int id = (to_sensor_dev_attr(attr))->index - 1; + return sprintf(buf, "CPU %d Temperature\n", id); +} + +static ssize_t get_cpu_temp(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int id = (to_sensor_dev_attr(attr))->index - 1; + int value = loongson3_cpu_temp(id); + return sprintf(buf, "%d\n", value); +} + +static SENSOR_DEVICE_ATTR(temp1_input, 0444, get_cpu_temp, NULL, 1); +static SENSOR_DEVICE_ATTR(temp1_label, 0444, cpu_temp_label, NULL, 1); +static SENSOR_DEVICE_ATTR(temp2_input, 0444, get_cpu_temp, NULL, 2); +static SENSOR_DEVICE_ATTR(temp2_label, 0444, cpu_temp_label, NULL, 2); +static SENSOR_DEVICE_ATTR(temp3_input, 0444, get_cpu_temp, NULL, 3); +static SENSOR_DEVICE_ATTR(temp3_label, 0444, cpu_temp_label, NULL, 3); +static SENSOR_DEVICE_ATTR(temp4_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp4_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp5_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp5_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp7_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp7_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp8_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp8_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp9_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp9_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp10_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp10_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp11_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp11_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp12_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp12_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp13_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp13_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp14_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp14_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp15_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp15_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp16_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp16_label, 0444, cpu_temp_label, NULL, 4); + +static struct attribute *cpu_hwmon_attributes[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_label.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp2_label.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp3_label.dev_attr.attr, + &sensor_dev_attr_temp4_input.dev_attr.attr, + &sensor_dev_attr_temp4_label.dev_attr.attr, + &sensor_dev_attr_temp5_input.dev_attr.attr, + &sensor_dev_attr_temp5_label.dev_attr.attr, + &sensor_dev_attr_temp6_input.dev_attr.attr, + &sensor_dev_attr_temp6_label.dev_attr.attr, + &sensor_dev_attr_temp7_input.dev_attr.attr, + &sensor_dev_attr_temp7_label.dev_attr.attr, + &sensor_dev_attr_temp8_input.dev_attr.attr, + &sensor_dev_attr_temp8_label.dev_attr.attr, + &sensor_dev_attr_temp9_input.dev_attr.attr, + &sensor_dev_attr_temp9_label.dev_attr.attr, + &sensor_dev_attr_temp10_input.dev_attr.attr, + &sensor_dev_attr_temp10_label.dev_attr.attr, + &sensor_dev_attr_temp11_input.dev_attr.attr, + &sensor_dev_attr_temp11_label.dev_attr.attr, + &sensor_dev_attr_temp12_input.dev_attr.attr, + &sensor_dev_attr_temp12_label.dev_attr.attr, + &sensor_dev_attr_temp13_input.dev_attr.attr, + &sensor_dev_attr_temp13_label.dev_attr.attr, + &sensor_dev_attr_temp14_input.dev_attr.attr, + &sensor_dev_attr_temp14_label.dev_attr.attr, + &sensor_dev_attr_temp15_input.dev_attr.attr, + &sensor_dev_attr_temp15_label.dev_attr.attr, + &sensor_dev_attr_temp16_input.dev_attr.attr, + &sensor_dev_attr_temp16_label.dev_attr.attr, + NULL +}; +static umode_t cpu_hwmon_is_visible(struct kobject *kobj, + struct attribute *attr, int i) +{ + int id = i / 2; + + if (id < nr_packages) + return attr->mode; + return 0; +} + +static struct attribute_group cpu_hwmon_group = { + .attrs = cpu_hwmon_attributes, + .is_visible = cpu_hwmon_is_visible, +}; + +static const struct attribute_group *cpu_hwmon_groups[] = { + &cpu_hwmon_group, + NULL +}; + +static int cpu_initial_threshold = 72000; +static int cpu_thermal_threshold = 96000; +module_param(cpu_thermal_threshold, int, 0644); +MODULE_PARM_DESC(cpu_thermal_threshold, "cpu thermal threshold (96000 (default))"); + +static struct delayed_work thermal_work; + +static void do_thermal_timer(struct work_struct *work) +{ + int i, value, temp_max = 0; + + for (i=0; i temp_max) + temp_max = value; + } + + if (temp_max <= cpu_thermal_threshold) + schedule_delayed_work(&thermal_work, msecs_to_jiffies(5000)); + else + orderly_poweroff(true); +} + +static int __init loongson_hwmon_init(void) +{ + int i, value, temp_max = 0; + + pr_info("Loongson Hwmon Enter...\n"); + + nr_packages = loongson_sysconf.nr_cpus / + loongson_sysconf.cores_per_package; + + cpu_hwmon_dev = hwmon_device_register_with_groups(NULL, "cpu_hwmon", + NULL, cpu_hwmon_groups); + if (IS_ERR(cpu_hwmon_dev)) { + pr_err("Hwmon register fail with %ld!\n", PTR_ERR(cpu_hwmon_dev)); + return PTR_ERR(cpu_hwmon_dev); + } + + for (i = 0; i < nr_packages; i++) { + value = loongson3_cpu_temp(i); + if (value > temp_max) + temp_max = value; + } + + pr_info("Initial CPU temperature is %d (highest).\n", temp_max); + if (temp_max > cpu_initial_threshold) + cpu_thermal_threshold += temp_max - cpu_initial_threshold; + + INIT_DEFERRABLE_WORK(&thermal_work, do_thermal_timer); + schedule_delayed_work(&thermal_work, msecs_to_jiffies(20000)); + + return 0; +} + +static void __exit loongson_hwmon_exit(void) +{ + cancel_delayed_work_sync(&thermal_work); + hwmon_device_unregister(cpu_hwmon_dev); +} + +module_init(loongson_hwmon_init); +module_exit(loongson_hwmon_exit); + +MODULE_AUTHOR("Yu Xiang "); +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("Loongson CPU Hwmon driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c new file mode 100644 index 0000000000000000000000000000000000000000..26b136a5d5e5e20dffe889da17171492b6e3ad7e --- /dev/null +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generic Loongson processor based LAPTOP/ALL-IN-ONE driver + * + * Jianmin Lv + * Huacai Chen + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* 1. Driver-wide structs and misc. variables */ + +/* ACPI HIDs */ +#define LOONGSON_ACPI_EC_HID "PNP0C09" +#define LOONGSON_ACPI_HKEY_HID "LOON0000" + +#define ACPI_LAPTOP_NAME "loongson-laptop" +#define ACPI_LAPTOP_ACPI_EVENT_PREFIX "loongson" + +#define MAX_ACPI_ARGS 3 +#define GENERIC_HOTKEY_MAP_MAX 64 + +#define GENERIC_EVENT_TYPE_OFF 12 +#define GENERIC_EVENT_TYPE_MASK 0xF000 +#define GENERIC_EVENT_CODE_MASK 0x0FFF + +struct generic_sub_driver { + u32 type; + char *name; + acpi_handle *handle; + struct acpi_device *device; + struct platform_driver *driver; + int (*init)(struct generic_sub_driver *sub_driver); + void (*notify)(struct generic_sub_driver *sub_driver, u32 event); + u8 acpi_notify_installed; +}; + +static u32 input_device_registered; +static struct input_dev *generic_inputdev; + +static acpi_handle hotkey_handle; +static struct key_entry hotkey_keycode_map[GENERIC_HOTKEY_MAP_MAX]; + +int loongson_laptop_turn_on_backlight(void); +int loongson_laptop_turn_off_backlight(void); +static int loongson_laptop_backlight_update(struct backlight_device *bd); + +/* 2. ACPI Helpers and device model */ + +static int acpi_evalf(acpi_handle handle, int *res, char *method, char *fmt, ...) +{ + char res_type; + char *fmt0 = fmt; + va_list ap; + int success, quiet; + acpi_status status; + struct acpi_object_list params; + struct acpi_buffer result, *resultp; + union acpi_object in_objs[MAX_ACPI_ARGS], out_obj; + + if (!*fmt) { + pr_err("acpi_evalf() called with empty format\n"); + return 0; + } + + if (*fmt == 'q') { + quiet = 1; + fmt++; + } else + quiet = 0; + + res_type = *(fmt++); + + params.count = 0; + params.pointer = &in_objs[0]; + + va_start(ap, fmt); + while (*fmt) { + char c = *(fmt++); + switch (c) { + case 'd': /* int */ + in_objs[params.count].integer.value = va_arg(ap, int); + in_objs[params.count++].type = ACPI_TYPE_INTEGER; + break; + /* add more types as needed */ + default: + pr_err("acpi_evalf() called with invalid format character '%c'\n", c); + va_end(ap); + return 0; + } + } + va_end(ap); + + if (res_type != 'v') { + result.length = sizeof(out_obj); + result.pointer = &out_obj; + resultp = &result; + } else + resultp = NULL; + + status = acpi_evaluate_object(handle, method, ¶ms, resultp); + + switch (res_type) { + case 'd': /* int */ + success = (status == AE_OK && out_obj.type == ACPI_TYPE_INTEGER); + if (success && res) + *res = out_obj.integer.value; + break; + case 'v': /* void */ + success = status == AE_OK; + break; + /* add more types as needed */ + default: + pr_err("acpi_evalf() called with invalid format character '%c'\n", res_type); + return 0; + } + + if (!success && !quiet) + pr_err("acpi_evalf(%s, %s, ...) failed: %s\n", + method, fmt0, acpi_format_exception(status)); + + return success; +} + +static int hotkey_status_get(int *status) +{ + if (!acpi_evalf(hotkey_handle, status, "GSWS", "d")) + return -EIO; + + return 0; +} + +static void dispatch_acpi_notify(acpi_handle handle, u32 event, void *data) +{ + struct generic_sub_driver *sub_driver = data; + + if (!sub_driver || !sub_driver->notify) + return; + sub_driver->notify(sub_driver, event); +} + +static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver) +{ + int rc; + acpi_status status; + + if (!*sub_driver->handle) + return 0; + + rc = acpi_bus_get_device(*sub_driver->handle, &sub_driver->device); + if (rc < 0) { + pr_err("acpi_bus_get_device(%s) failed: %d\n", sub_driver->name, rc); + return -ENODEV; + } + + sub_driver->device->driver_data = sub_driver; + sprintf(acpi_device_class(sub_driver->device), "%s/%s", + ACPI_LAPTOP_ACPI_EVENT_PREFIX, sub_driver->name); + + status = acpi_install_notify_handler(*sub_driver->handle, + sub_driver->type, dispatch_acpi_notify, sub_driver); + if (ACPI_FAILURE(status)) { + if (status == AE_ALREADY_EXISTS) { + pr_notice("Another device driver is already " + "handling %s events\n", sub_driver->name); + } else { + pr_err("acpi_install_notify_handler(%s) failed: %s\n", + sub_driver->name, acpi_format_exception(status)); + } + return -ENODEV; + } + sub_driver->acpi_notify_installed = 1; + + return 0; +} + +#ifdef CONFIG_PM +static int loongson_hotkey_suspend(struct device *dev) +{ + return 0; +} + +static int loongson_hotkey_resume(struct device *dev) +{ + int status = 0; + struct key_entry ke; + struct backlight_device *bd; + + bd = backlight_device_get_by_type(BACKLIGHT_PLATFORM); + if (bd) { + loongson_laptop_backlight_update(bd) ? + pr_warn("Loongson_backlight: resume brightness failed") : + pr_info("Loongson_backlight: resume brightness %d\n", bd->props.brightness); + } + + /* + * Only if the firmware supports SW_LID event model, we can handle the + * event. This is for the consideration of development board without EC. + */ + if (test_bit(SW_LID, generic_inputdev->swbit)) { + if (hotkey_status_get(&status) < 0) + return -EIO; + /* + * The input device sw element records the last lid status. + * When the system is awakened by other wake-up sources, + * the lid event will also be reported. The judgment of + * adding SW_LID bit which in sw element can avoid this + * case. + * + * Input system will drop lid event when current lid event + * value and last lid status in the same. So laptop driver + * doesn't report repeated events. + * + * Lid status is generally 0, but hardware exception is + * considered. So add lid status confirmation. + */ + if (test_bit(SW_LID, generic_inputdev->sw) && !(status & (1 << SW_LID))) { + ke.type = KE_SW; + ke.sw.value = (u8)status; + ke.sw.code = SW_LID; + sparse_keymap_report_entry(generic_inputdev, &ke, 1, true); + } + } + + return 0; +} + +static SIMPLE_DEV_PM_OPS(loongson_hotkey_pm, + loongson_hotkey_suspend, loongson_hotkey_resume); +#endif + +static int loongson_hotkey_probe(struct platform_device *pdev) +{ + hotkey_handle = ACPI_HANDLE(&pdev->dev); + + if (!hotkey_handle) + return -ENODEV; + + return 0; +} + +static const struct acpi_device_id loongson_device_ids[] = { + {LOONGSON_ACPI_HKEY_HID, 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, loongson_device_ids); + +static struct platform_driver loongson_hotkey_driver = { + .probe = loongson_hotkey_probe, + .driver = { + .name = "loongson-hotkey", + .owner = THIS_MODULE, + .pm = pm_ptr(&loongson_hotkey_pm), + .acpi_match_table = loongson_device_ids, + }, +}; + +static int hotkey_map(void) +{ + u32 index; + acpi_status status; + struct acpi_buffer buf; + union acpi_object *pack; + + buf.length = ACPI_ALLOCATE_BUFFER; + status = acpi_evaluate_object_typed(hotkey_handle, "KMAP", NULL, &buf, ACPI_TYPE_PACKAGE); + if (status != AE_OK) { + pr_err("ACPI exception: %s\n", acpi_format_exception(status)); + return -1; + } + pack = buf.pointer; + for (index = 0; index < pack->package.count; index++) { + union acpi_object *element, *sub_pack; + + sub_pack = &pack->package.elements[index]; + + element = &sub_pack->package.elements[0]; + hotkey_keycode_map[index].type = element->integer.value; + element = &sub_pack->package.elements[1]; + hotkey_keycode_map[index].code = element->integer.value; + element = &sub_pack->package.elements[2]; + hotkey_keycode_map[index].keycode = element->integer.value; + } + + return 0; +} + +static int hotkey_backlight_set(bool enable) +{ + if (!acpi_evalf(hotkey_handle, NULL, "VCBL", "vd", enable ? 1 : 0)) + return -EIO; + + return 0; +} + +static int ec_get_brightness(void) +{ + int status = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, &status, "ECBG", "d")) + return -EIO; + + return status; +} + +static int ec_set_brightness(int level) +{ + + int ret = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, NULL, "ECBS", "vd", level)) + ret = -EIO; + + return ret; +} + +static int ec_backlight_level(u8 level) +{ + int status = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d")) + return -EIO; + + if ((status < 0) || (level > status)) + return status; + + if (!acpi_evalf(hotkey_handle, &status, "ECSL", "d")) + return -EIO; + + if ((status < 0) || (level < status)) + return status; + + return level; +} + +static int loongson_laptop_backlight_update(struct backlight_device *bd) +{ + int lvl = ec_backlight_level(bd->props.brightness); + if (lvl < 0) + return -EIO; + if (ec_set_brightness(lvl)) + return -EIO; + + return 0; +} + +static int loongson_laptop_get_brightness(struct backlight_device *bd) +{ + int level; + + level = ec_get_brightness(); + if (level < 0) + return -EIO; + + return level; +} + +static const struct backlight_ops backlight_laptop_ops = { + .update_status = loongson_laptop_backlight_update, + .get_brightness = loongson_laptop_get_brightness, +}; + +static int laptop_backlight_register(void) +{ + int status = 0; + struct backlight_properties props; + + memset(&props, 0, sizeof(props)); + + if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d")) + return -EIO; + + props.brightness = 1; + props.max_brightness = status; + props.type = BACKLIGHT_PLATFORM; + + backlight_device_register("loongson_laptop", + NULL, NULL, &backlight_laptop_ops, &props); + + return 0; +} + +int loongson_laptop_turn_on_backlight(void) +{ + int status; + union acpi_object arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list args = { 1, &arg0 }; + + arg0.integer.value = 1; + status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); + if (ACPI_FAILURE(status)) { + pr_info("Loongson lvds error:0x%x\n", status); + return -ENODEV; + } + return 0; +} + +int loongson_laptop_turn_off_backlight(void) +{ + int status; + union acpi_object arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list args = { 1, &arg0 }; + + arg0.integer.value = 0; + status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); + if (ACPI_FAILURE(status)) { + pr_info("Loongson lvds error:0x%x\n", status); + return -ENODEV; + } + return 0; +} + +static int __init event_init(struct generic_sub_driver *sub_driver) +{ + int ret; + + ret = hotkey_map(); + if (ret < 0) { + pr_err("Failed to parse keymap from DSDT\n"); + return ret; + } + + ret = sparse_keymap_setup(generic_inputdev, hotkey_keycode_map, NULL); + if (ret < 0) { + pr_err("Failed to setup input device keymap\n"); + input_free_device(generic_inputdev); + generic_inputdev = NULL; + + return ret; + } + + /* + * This hotkey driver handle backlight event when + * acpi_video_get_backlight_type() gets acpi_backlight_vendor + */ + if (acpi_video_get_backlight_type() == acpi_backlight_vendor) + hotkey_backlight_set(true); + else + hotkey_backlight_set(false); + + pr_info("ACPI: enabling firmware HKEY event interface...\n"); + + return ret; +} + +static void event_notify(struct generic_sub_driver *sub_driver, u32 event) +{ + int type, scan_code; + struct key_entry *ke = NULL; + + scan_code = event & GENERIC_EVENT_CODE_MASK; + type = (event & GENERIC_EVENT_TYPE_MASK) >> GENERIC_EVENT_TYPE_OFF; + ke = sparse_keymap_entry_from_scancode(generic_inputdev, scan_code); + if (ke) { + if (type == KE_SW) { + int status = 0; + + if (hotkey_status_get(&status) < 0) + return; + ke->sw.value = !!(status & (1 << ke->sw.code)); + } + sparse_keymap_report_entry(generic_inputdev, ke, 1, true); + } +} + +/* 3. Infrastructure */ + +static void generic_subdriver_exit(struct generic_sub_driver *sub_driver); + +static int __init generic_subdriver_init(struct generic_sub_driver *sub_driver) +{ + int ret; + + if (!sub_driver || !sub_driver->driver) + return -EINVAL; + + ret = platform_driver_register(sub_driver->driver); + if (ret) + return -EINVAL; + + if (sub_driver->init) { + ret = sub_driver->init(sub_driver); + if (ret) + goto err_out; + } + + if (sub_driver->notify) { + ret = setup_acpi_notify(sub_driver); + if (ret == -ENODEV) { + ret = 0; + goto err_out; + } + if (ret < 0) + goto err_out; + } + + return 0; + +err_out: + generic_subdriver_exit(sub_driver); + return ret; +} + +static void generic_subdriver_exit(struct generic_sub_driver *sub_driver) +{ + + if (sub_driver->acpi_notify_installed) { + acpi_remove_notify_handler(*sub_driver->handle, + sub_driver->type, dispatch_acpi_notify); + sub_driver->acpi_notify_installed = 0; + } + platform_driver_unregister(sub_driver->driver); +} + +static struct generic_sub_driver generic_sub_drivers[] __refdata = { + { + .name = "hotkey", + .init = event_init, + .notify = event_notify, + .handle = &hotkey_handle, + .type = ACPI_DEVICE_NOTIFY, + .driver = &loongson_hotkey_driver, + }, +}; + +static int __init generic_acpi_laptop_init(void) +{ + bool ec_found; + int i, ret, status; + + if (acpi_disabled) + return -ENODEV; + + /* The EC device is required */ + ec_found = acpi_dev_found(LOONGSON_ACPI_EC_HID); + if (!ec_found) + return -ENODEV; + + /* Enable SCI for EC */ + acpi_write_bit_register(ACPI_BITREG_SCI_ENABLE, 1); + + generic_inputdev = input_allocate_device(); + if (!generic_inputdev) { + pr_err("Unable to allocate input device\n"); + return -ENOMEM; + } + + /* Prepare input device, but don't register */ + generic_inputdev->name = + "Loongson Generic Laptop/All-in-One Extra Buttons"; + generic_inputdev->phys = ACPI_LAPTOP_NAME "/input0"; + generic_inputdev->id.bustype = BUS_HOST; + generic_inputdev->dev.parent = NULL; + + /* Init subdrivers */ + for (i = 0; i < ARRAY_SIZE(generic_sub_drivers); i++) { + ret = generic_subdriver_init(&generic_sub_drivers[i]); + if (ret < 0) { + input_free_device(generic_inputdev); + while (--i >= 0) + generic_subdriver_exit(&generic_sub_drivers[i]); + return ret; + } + } + + ret = input_register_device(generic_inputdev); + if (ret < 0) { + input_free_device(generic_inputdev); + while (--i >= 0) + generic_subdriver_exit(&generic_sub_drivers[i]); + pr_err("Unable to register input device\n"); + return ret; + } + + input_device_registered = 1; + + if (acpi_evalf(hotkey_handle, &status, "ECBG", "d")) { + pr_info("Loongson Laptop used, init brightness is 0x%x\n", status); + ret = laptop_backlight_register(); + if (ret < 0) + pr_err("Loongson Laptop: laptop-backlight device register failed\n"); + } + + return 0; +} + +static void __exit generic_acpi_laptop_exit(void) +{ + if (generic_inputdev) { + if (input_device_registered) + input_unregister_device(generic_inputdev); + else + input_free_device(generic_inputdev); + } +} + +module_init(generic_acpi_laptop_init); +module_exit(generic_acpi_laptop_exit); + +MODULE_AUTHOR("Jianmin Lv "); +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("Loongson Laptop/All-in-One ACPI Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pnp/pnpacpi/rsparser.c b/drivers/pnp/pnpacpi/rsparser.c index da78dc77aed32e4fbd589d138f13fa3cdcbba4d8..9879deb4dc0b57002920e6469567bc1505e3d03b 100644 --- a/drivers/pnp/pnpacpi/rsparser.c +++ b/drivers/pnp/pnpacpi/rsparser.c @@ -151,13 +151,13 @@ static int vendor_resource_matches(struct pnp_dev *dev, static void pnpacpi_parse_allocated_vendor(struct pnp_dev *dev, struct acpi_resource_vendor_typed *vendor) { - if (vendor_resource_matches(dev, vendor, &hp_ccsr_uuid, 16)) { - u64 start, length; + struct { u64 start, length; } range; - memcpy(&start, vendor->byte_data, sizeof(start)); - memcpy(&length, vendor->byte_data + 8, sizeof(length)); - - pnp_add_mem_resource(dev, start, start + length - 1, 0); + if (vendor_resource_matches(dev, vendor, &hp_ccsr_uuid, + sizeof(range))) { + memcpy(&range, vendor->byte_data, sizeof(range)); + pnp_add_mem_resource(dev, range.start, range.start + + range.length - 1, 0); } } diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c index de1fa71be1e839c1090a1ed5e166638ecb51d66d..d1071dbb904ee5809414b0cb8fabbac2ac761250 100644 --- a/drivers/power/supply/cw2015_battery.c +++ b/drivers/power/supply/cw2015_battery.c @@ -490,7 +490,7 @@ static int cw_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: if (cw_battery_valid_time_to_empty(cw_bat)) - val->intval = cw_bat->time_to_empty; + val->intval = cw_bat->time_to_empty * 60; else val->intval = 0; break; diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 00c642fa2eed15266e56f375b1fc2008aac1d986..db951a37c0488ec6e2cbae9117336eda3a23389e 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -60,9 +60,10 @@ static int jz4740_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) snprintf(name, sizeof(name), "timer%u", pwm->hwpwm); clk = clk_get(chip->dev, name); - if (IS_ERR(clk)) - return dev_err_probe(chip->dev, PTR_ERR(clk), - "Failed to get clock\n"); + if (IS_ERR(clk)) { + dev_err(chip->dev, "error %pe: Failed to get clock\n", clk); + return PTR_ERR(clk); + } err = clk_prepare_enable(clk); if (err < 0) { diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index d3be944f2ae96dbf3313ec593956ca440f1eb8c3..69b7bc604946625befcf35ee27f1181ef4f1de91 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -115,14 +115,14 @@ static int stm32_pwm_raw_capture(struct stm32_pwm *priv, struct pwm_device *pwm, int ret; /* Ensure registers have been updated, enable counter and capture */ - regmap_update_bits(priv->regmap, TIM_EGR, TIM_EGR_UG, TIM_EGR_UG); - regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, TIM_CR1_CEN); + regmap_set_bits(priv->regmap, TIM_EGR, TIM_EGR_UG); + regmap_set_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN); /* Use cc1 or cc3 DMA resp for PWM input channels 1 & 2 or 3 & 4 */ dma_id = pwm->hwpwm < 2 ? STM32_TIMERS_DMA_CH1 : STM32_TIMERS_DMA_CH3; ccen = pwm->hwpwm < 2 ? TIM_CCER_CC12E : TIM_CCER_CC34E; ccr = pwm->hwpwm < 2 ? TIM_CCR1 : TIM_CCR3; - regmap_update_bits(priv->regmap, TIM_CCER, ccen, ccen); + regmap_set_bits(priv->regmap, TIM_CCER, ccen); /* * Timer DMA burst mode. Request 2 registers, 2 bursts, to get both @@ -160,8 +160,8 @@ static int stm32_pwm_raw_capture(struct stm32_pwm *priv, struct pwm_device *pwm, } stop: - regmap_update_bits(priv->regmap, TIM_CCER, ccen, 0); - regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, 0); + regmap_clear_bits(priv->regmap, TIM_CCER, ccen); + regmap_clear_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN); return ret; } @@ -359,7 +359,7 @@ static int stm32_pwm_config(struct stm32_pwm *priv, int ch, regmap_write(priv->regmap, TIM_PSC, prescaler); regmap_write(priv->regmap, TIM_ARR, prd - 1); - regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_ARPE, TIM_CR1_ARPE); + regmap_set_bits(priv->regmap, TIM_CR1, TIM_CR1_ARPE); /* Calculate the duty cycles */ dty = prd * duty_ns; @@ -377,7 +377,7 @@ static int stm32_pwm_config(struct stm32_pwm *priv, int ch, else regmap_update_bits(priv->regmap, TIM_CCMR2, mask, ccmr); - regmap_update_bits(priv->regmap, TIM_BDTR, TIM_BDTR_MOE, TIM_BDTR_MOE); + regmap_set_bits(priv->regmap, TIM_BDTR, TIM_BDTR_MOE); return 0; } @@ -411,13 +411,13 @@ static int stm32_pwm_enable(struct stm32_pwm *priv, int ch) if (priv->have_complementary_output) mask |= TIM_CCER_CC1NE << (ch * 4); - regmap_update_bits(priv->regmap, TIM_CCER, mask, mask); + regmap_set_bits(priv->regmap, TIM_CCER, mask); /* Make sure that registers are updated */ - regmap_update_bits(priv->regmap, TIM_EGR, TIM_EGR_UG, TIM_EGR_UG); + regmap_set_bits(priv->regmap, TIM_EGR, TIM_EGR_UG); /* Enable controller */ - regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, TIM_CR1_CEN); + regmap_set_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN); return 0; } @@ -431,11 +431,11 @@ static void stm32_pwm_disable(struct stm32_pwm *priv, int ch) if (priv->have_complementary_output) mask |= TIM_CCER_CC1NE << (ch * 4); - regmap_update_bits(priv->regmap, TIM_CCER, mask, 0); + regmap_clear_bits(priv->regmap, TIM_CCER, mask); /* When all channels are disabled, we can disable the controller */ if (!active_channels(priv)) - regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN, 0); + regmap_clear_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN); clk_disable(priv->clk); } @@ -568,41 +568,30 @@ static void stm32_pwm_detect_complementary(struct stm32_pwm *priv) * If complementary bit doesn't exist writing 1 will have no * effect so we can detect it. */ - regmap_update_bits(priv->regmap, - TIM_CCER, TIM_CCER_CC1NE, TIM_CCER_CC1NE); + regmap_set_bits(priv->regmap, TIM_CCER, TIM_CCER_CC1NE); regmap_read(priv->regmap, TIM_CCER, &ccer); - regmap_update_bits(priv->regmap, TIM_CCER, TIM_CCER_CC1NE, 0); + regmap_clear_bits(priv->regmap, TIM_CCER, TIM_CCER_CC1NE); priv->have_complementary_output = (ccer != 0); } -static int stm32_pwm_detect_channels(struct stm32_pwm *priv) +static unsigned int stm32_pwm_detect_channels(struct stm32_pwm *priv, + unsigned int *num_enabled) { - u32 ccer; - int npwm = 0; + u32 ccer, ccer_backup; /* * If channels enable bits don't exist writing 1 will have no * effect so we can detect and count them. */ - regmap_update_bits(priv->regmap, - TIM_CCER, TIM_CCER_CCXE, TIM_CCER_CCXE); + regmap_read(priv->regmap, TIM_CCER, &ccer_backup); + regmap_set_bits(priv->regmap, TIM_CCER, TIM_CCER_CCXE); regmap_read(priv->regmap, TIM_CCER, &ccer); - regmap_update_bits(priv->regmap, TIM_CCER, TIM_CCER_CCXE, 0); - - if (ccer & TIM_CCER_CC1E) - npwm++; + regmap_write(priv->regmap, TIM_CCER, ccer_backup); - if (ccer & TIM_CCER_CC2E) - npwm++; + *num_enabled = hweight32(ccer_backup & TIM_CCER_CCXE); - if (ccer & TIM_CCER_CC3E) - npwm++; - - if (ccer & TIM_CCER_CC4E) - npwm++; - - return npwm; + return hweight32(ccer & TIM_CCER_CCXE); } static int stm32_pwm_probe(struct platform_device *pdev) @@ -611,6 +600,8 @@ static int stm32_pwm_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct stm32_timers *ddata = dev_get_drvdata(pdev->dev.parent); struct stm32_pwm *priv; + unsigned int num_enabled; + unsigned int i; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -636,7 +627,11 @@ static int stm32_pwm_probe(struct platform_device *pdev) priv->chip.base = -1; priv->chip.dev = dev; priv->chip.ops = &stm32pwm_ops; - priv->chip.npwm = stm32_pwm_detect_channels(priv); + priv->chip.npwm = stm32_pwm_detect_channels(priv, &num_enabled); + + /* Initialize clock refcount to number of enabled PWM channels. */ + for (i = 0; i < num_enabled; i++) + clk_enable(priv->clk); ret = pwmchip_add(&priv->chip); if (ret < 0) diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 50cbfe54c7db04d54f3a46fa21ceac0ea81cf9a0..96928a832a4373e894600f1c024676079546bcef 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -2770,7 +2770,8 @@ static int _regulator_enable(struct regulator *regulator) /* Fallthrough on positive return values - already enabled */ } - rdev->use_count++; + if (regulator->enable_count == 1) + rdev->use_count++; return 0; @@ -2848,37 +2849,40 @@ static int _regulator_disable(struct regulator *regulator) lockdep_assert_held_once(&rdev->mutex.base); - if (WARN(rdev->use_count <= 0, + if (WARN(regulator->enable_count == 0, "unbalanced disables for %s\n", rdev_get_name(rdev))) return -EIO; - /* are we the last user and permitted to disable ? */ - if (rdev->use_count == 1 && - (rdev->constraints && !rdev->constraints->always_on)) { - - /* we are last user */ - if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) { - ret = _notifier_call_chain(rdev, - REGULATOR_EVENT_PRE_DISABLE, - NULL); - if (ret & NOTIFY_STOP_MASK) - return -EINVAL; - - ret = _regulator_do_disable(rdev); - if (ret < 0) { - rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret)); - _notifier_call_chain(rdev, - REGULATOR_EVENT_ABORT_DISABLE, + if (regulator->enable_count == 1) { + /* disabling last enable_count from this regulator */ + /* are we the last user and permitted to disable ? */ + if (rdev->use_count == 1 && + (rdev->constraints && !rdev->constraints->always_on)) { + + /* we are last user */ + if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) { + ret = _notifier_call_chain(rdev, + REGULATOR_EVENT_PRE_DISABLE, + NULL); + if (ret & NOTIFY_STOP_MASK) + return -EINVAL; + + ret = _regulator_do_disable(rdev); + if (ret < 0) { + rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret)); + _notifier_call_chain(rdev, + REGULATOR_EVENT_ABORT_DISABLE, + NULL); + return ret; + } + _notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE, NULL); - return ret; } - _notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE, - NULL); - } - rdev->use_count = 0; - } else if (rdev->use_count > 1) { - rdev->use_count--; + rdev->use_count = 0; + } else if (rdev->use_count > 1) { + rdev->use_count--; + } } if (ret == 0) diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 7d7ed4e5cce7b3d0cb281414f40e4d4694111b6c..a8b5d10880c0d2d38216c03574e85f52c34039aa 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -387,6 +387,7 @@ static void virtio_rpmsg_release_device(struct device *dev) struct rpmsg_device *rpdev = to_rpmsg_device(dev); struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + kfree(rpdev->driver_override); kfree(vch); } diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 7f560937bf7cb049d0323e80e58a311d30ad43d7..2c4ccab6e462d9379833a8d0a580c1b773c061d6 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -292,7 +292,7 @@ static int cmos_read_alarm(struct device *dev, struct rtc_wkalrm *t) /* This not only a rtc_op, but also called directly */ if (!is_valid_irq(cmos->irq)) - return -EIO; + return -ETIMEDOUT; /* Basic alarms only support hour, minute, and seconds fields. * Some also support day and month, for alarms up to a year in @@ -557,7 +557,7 @@ static int cmos_set_alarm(struct device *dev, struct rtc_wkalrm *t) * Use mc146818_avoid_UIP() to avoid this. */ if (!mc146818_avoid_UIP(cmos_set_alarm_callback, &p)) - return -EIO; + return -ETIMEDOUT; cmos->alarm_expires = rtc_tm_to_time64(&t->time); diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h index 5d054d5c70a59e82c5d6e51f405ab7037f589c6a..f2e587e66e19dad62254e120bd15dfa41b723086 100644 --- a/drivers/scsi/arcmsr/arcmsr.h +++ b/drivers/scsi/arcmsr/arcmsr.h @@ -77,9 +77,13 @@ struct device_attribute; #ifndef PCI_DEVICE_ID_ARECA_1203 #define PCI_DEVICE_ID_ARECA_1203 0x1203 #endif +#ifndef PCI_DEVICE_ID_ARECA_1883 +#define PCI_DEVICE_ID_ARECA_1883 0x1883 +#endif #ifndef PCI_DEVICE_ID_ARECA_1884 #define PCI_DEVICE_ID_ARECA_1884 0x1884 #endif +#define PCI_DEVICE_ID_ARECA_1886_0 0x1886 #define PCI_DEVICE_ID_ARECA_1886 0x188A #define ARCMSR_HOURS (1000 * 60 * 60 * 4) #define ARCMSR_MINUTES (1000 * 60 * 60) diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index 9294a2c677b3ef9146c3d103313d936fe71e07d6..199b102f31a24d5577552895b9c9406211eabb07 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -208,8 +208,12 @@ static struct pci_device_id arcmsr_device_id_table[] = { .driver_data = ACB_ADAPTER_TYPE_A}, {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1880), .driver_data = ACB_ADAPTER_TYPE_C}, + {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1883), + .driver_data = ACB_ADAPTER_TYPE_C}, {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1884), .driver_data = ACB_ADAPTER_TYPE_E}, + {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1886_0), + .driver_data = ACB_ADAPTER_TYPE_F}, {PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1886), .driver_data = ACB_ADAPTER_TYPE_F}, {0, 0}, /* Terminating entry */ @@ -4701,9 +4705,11 @@ static const char *arcmsr_info(struct Scsi_Host *host) case PCI_DEVICE_ID_ARECA_1680: case PCI_DEVICE_ID_ARECA_1681: case PCI_DEVICE_ID_ARECA_1880: + case PCI_DEVICE_ID_ARECA_1883: case PCI_DEVICE_ID_ARECA_1884: type = "SAS/SATA"; break; + case PCI_DEVICE_ID_ARECA_1886_0: case PCI_DEVICE_ID_ARECA_1886: type = "NVMe/SAS/SATA"; break; diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c index 7cfeb6886237c0f44071fccc1ec7480b80679ec3..61c12dde967eea5d51e3f2580937b9c653b16455 100644 --- a/drivers/scsi/libfc/fc_fcp.c +++ b/drivers/scsi/libfc/fc_fcp.c @@ -270,6 +270,11 @@ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp) if (!fsp->seq_ptr) return -EINVAL; + if (fsp->state & FC_SRB_ABORT_PENDING) { + FC_FCP_DBG(fsp, "abort already pending\n"); + return -EBUSY; + } + per_cpu_ptr(fsp->lp->stats, get_cpu())->FcpPktAborts++; put_cpu(); @@ -1681,7 +1686,7 @@ static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp) if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY) fc_fcp_rec(fsp); else - fc_fcp_recovery(fsp, FC_ERROR); + fc_fcp_recovery(fsp, FC_TIMED_OUT); break; } fc_fcp_unlock_pkt(fsp); @@ -1700,11 +1705,12 @@ static void fc_fcp_recovery(struct fc_fcp_pkt *fsp, u8 code) fsp->status_code = code; fsp->cdb_status = 0; fsp->io_status = 0; - /* - * if this fails then we let the scsi command timer fire and - * scsi-ml escalate. - */ - fc_fcp_send_abort(fsp); + if (!fsp->cmd) + /* + * Only abort non-scsi commands; otherwise let the + * scsi command timer fire and scsi-ml escalate. + */ + fc_fcp_send_abort(fsp); } /** diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 03bc472f302a290607cfde8ac7cddb3bd6389776..cf69f831a72530e4d7b5a8a5b638ea861d40d817 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -32,6 +32,7 @@ struct lpfc_sli2_slim; #define ELX_MODEL_NAME_SIZE 80 +#define ELX_FW_NAME_SIZE 84 #define LPFC_PCI_DEV_LP 0x1 #define LPFC_PCI_DEV_OC 0x2 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 1bb3c96a04bd6f5165cb10e2dd7b1cf7cd956fc0..5f2009327a593bad8b78a0a7e4ad5202ead5cd9f 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -13026,7 +13026,7 @@ lpfc_write_firmware(const struct firmware *fw, void *context) int lpfc_sli4_request_firmware_update(struct lpfc_hba *phba, uint8_t fw_upgrade) { - uint8_t file_name[ELX_MODEL_NAME_SIZE]; + char file_name[ELX_FW_NAME_SIZE] = {0}; int ret; const struct firmware *fw; @@ -13035,7 +13035,7 @@ lpfc_sli4_request_firmware_update(struct lpfc_hba *phba, uint8_t fw_upgrade) LPFC_SLI_INTF_IF_TYPE_2) return -EPERM; - snprintf(file_name, ELX_MODEL_NAME_SIZE, "%s.grp", phba->ModelName); + scnprintf(file_name, sizeof(file_name), "%s.grp", phba->ModelName); if (fw_upgrade == INT_FW_UPGRADE) { ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG, diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c index 2c734ea0784b761610cabc5cedce1ebdd21e9436..898658ab1dcd4a631409d6b783026a9115c640a1 100644 --- a/drivers/spi/spi-bcm-qspi.c +++ b/drivers/spi/spi-bcm-qspi.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include "spi-bcm-qspi.h" @@ -1048,7 +1048,7 @@ static int bcm_qspi_exec_mem_op(struct spi_mem *mem, /* non-aligned and very short transfers are handled by MSPI */ if (!IS_ALIGNED((uintptr_t)addr, 4) || !IS_ALIGNED((uintptr_t)buf, 4) || - len < 4) + len < 4 || op->cmd.opcode == SPINOR_OP_RDSFDP) mspi_read = true; if (!has_bspi(qspi) || mspi_read) diff --git a/drivers/staging/media/rkisp1/rkisp1-dev.c b/drivers/staging/media/rkisp1/rkisp1-dev.c index 06de5540c8af4841ab47d109f389b32c23220541..663b2efec9b09ae9202057911c510ca6cc2823d1 100644 --- a/drivers/staging/media/rkisp1/rkisp1-dev.c +++ b/drivers/staging/media/rkisp1/rkisp1-dev.c @@ -518,7 +518,7 @@ static int rkisp1_probe(struct platform_device *pdev) ret = v4l2_device_register(rkisp1->dev, &rkisp1->v4l2_dev); if (ret) - return ret; + goto err_pm_runtime_disable; ret = media_device_register(&rkisp1->media_dev); if (ret) { @@ -538,6 +538,7 @@ static int rkisp1_probe(struct platform_device *pdev) media_device_unregister(&rkisp1->media_dev); err_unreg_v4l2_dev: v4l2_device_unregister(&rkisp1->v4l2_dev); +err_pm_runtime_disable: pm_runtime_disable(&pdev->dev); return ret; } diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index bd4118f1b694470b1ebbd3027e348d3aea8947ea..25765ebb756aed213ac2d34d53ad416fe15ddea5 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1497,7 +1497,7 @@ static int omap8250_remove(struct platform_device *pdev) err = pm_runtime_resume_and_get(&pdev->dev); if (err) - return err; + dev_err(&pdev->dev, "Failed to resume hardware\n"); serial8250_unregister_port(priv->line); priv->line = -ENODEV; diff --git a/drivers/tty/serial/8250/8250_pnp.c b/drivers/tty/serial/8250/8250_pnp.c index de90d681b64c30e84ee1aed96ee69f2a198e9fd8..e51421f97281687b651134d1f54444873a892f28 100644 --- a/drivers/tty/serial/8250/8250_pnp.c +++ b/drivers/tty/serial/8250/8250_pnp.c @@ -474,7 +474,9 @@ serial_pnp_probe(struct pnp_dev *dev, const struct pnp_device_id *dev_id) uart.port.flags |= UPF_SKIP_TEST | UPF_BOOT_AUTOCONF; if (pnp_irq_flags(dev, 0) & IORESOURCE_IRQ_SHAREABLE) uart.port.flags |= UPF_SHARE_IRQ; - uart.port.uartclk = 1843200; + if (device_property_read_u32(&dev->dev, "clock-frequency", &uart.port.uartclk)) { + uart.port.uartclk = 1843200; + } uart.port.dev = &dev->dev; line = serial8250_register_8250_port(&uart); diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index da63ef345e81120a4e22cb2a0cb2396295253bfd..2eca7b065c0a35839bf681a864b19c08feec3059 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -461,13 +461,13 @@ static void imx_uart_stop_tx(struct uart_port *port) ucr1 = imx_uart_readl(sport, UCR1); imx_uart_writel(sport, ucr1 & ~UCR1_TRDYEN, UCR1); + ucr4 = imx_uart_readl(sport, UCR4); usr2 = imx_uart_readl(sport, USR2); - if (!(usr2 & USR2_TXDC)) { + if ((!(usr2 & USR2_TXDC)) && (ucr4 & UCR4_TCEN)) { /* The shifter is still busy, so retry once TC triggers */ return; } - ucr4 = imx_uart_readl(sport, UCR4); ucr4 &= ~UCR4_TCEN; imx_uart_writel(sport, ucr4, UCR4); @@ -2346,7 +2346,7 @@ static int imx_uart_probe(struct platform_device *pdev) /* For register access, we only need to enable the ipg clock. */ ret = clk_prepare_enable(sport->clk_ipg); if (ret) { - dev_err(&pdev->dev, "failed to enable per clk: %d\n", ret); + dev_err(&pdev->dev, "failed to enable ipg clk: %d\n", ret); return ret; } @@ -2358,10 +2358,8 @@ static int imx_uart_probe(struct platform_device *pdev) sport->ufcr = readl(sport->port.membase + UFCR); ret = uart_get_rs485_mode(&sport->port); - if (ret) { - clk_disable_unprepare(sport->clk_ipg); - return ret; - } + if (ret) + goto err_clk; if (sport->port.rs485.flags & SER_RS485_ENABLED && (!sport->have_rtscts && !sport->have_rtsgpio)) @@ -2415,8 +2413,6 @@ static int imx_uart_probe(struct platform_device *pdev) imx_uart_writel(sport, ucr3, UCR3); } - clk_disable_unprepare(sport->clk_ipg); - hrtimer_init(&sport->trigger_start_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer_init(&sport->trigger_stop_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL); sport->trigger_start_tx.function = imx_trigger_start_tx; @@ -2432,7 +2428,7 @@ static int imx_uart_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "failed to request rx irq: %d\n", ret); - return ret; + goto err_clk; } ret = devm_request_irq(&pdev->dev, txirq, imx_uart_txint, 0, @@ -2440,7 +2436,7 @@ static int imx_uart_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "failed to request tx irq: %d\n", ret); - return ret; + goto err_clk; } ret = devm_request_irq(&pdev->dev, rtsirq, imx_uart_rtsint, 0, @@ -2448,14 +2444,14 @@ static int imx_uart_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "failed to request rts irq: %d\n", ret); - return ret; + goto err_clk; } } else { ret = devm_request_irq(&pdev->dev, rxirq, imx_uart_int, 0, dev_name(&pdev->dev), sport); if (ret) { dev_err(&pdev->dev, "failed to request irq: %d\n", ret); - return ret; + goto err_clk; } } @@ -2463,7 +2459,12 @@ static int imx_uart_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sport); - return uart_add_one_port(&imx_uart_uart_driver, &sport->port); + ret = uart_add_one_port(&imx_uart_uart_driver, &sport->port); + +err_clk: + clk_disable_unprepare(sport->clk_ipg); + + return ret; } static int imx_uart_remove(struct platform_device *pdev) diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index fd9be81bcfd86436e8049fd9a7c85d99dfada2ea..31e0c5c3ddeacb32a50298a33b66b65946d2040e 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #define SC16IS7XX_NAME "sc16is7xx" @@ -1449,9 +1450,12 @@ static int sc16is7xx_spi_probe(struct spi_device *spi) /* Setup SPI bus */ spi->bits_per_word = 8; - /* only supports mode 0 on SC16IS762 */ + /* For all variants, only mode 0 is supported */ + if ((spi->mode & SPI_MODE_X_MASK) != SPI_MODE_0) + return dev_err_probe(&spi->dev, -EINVAL, "Unsupported SPI mode\n"); + spi->mode = spi->mode ? : SPI_MODE_0; - spi->max_speed_hz = spi->max_speed_hz ? : 15000000; + spi->max_speed_hz = spi->max_speed_hz ? : 4 * HZ_PER_MHZ; ret = spi_setup(spi); if (ret) return ret; diff --git a/drivers/tty/tty.h b/drivers/tty/tty.h index 1908f27a795a089468e6f424a0b85480200da733..3d2d82ff6a034ea984221c0b8490626667d6acec 100644 --- a/drivers/tty/tty.h +++ b/drivers/tty/tty.h @@ -64,7 +64,7 @@ int tty_check_change(struct tty_struct *tty); void __stop_tty(struct tty_struct *tty); void __start_tty(struct tty_struct *tty); void tty_write_unlock(struct tty_struct *tty); -int tty_write_lock(struct tty_struct *tty, int ndelay); +int tty_write_lock(struct tty_struct *tty, bool ndelay); void tty_vhangup_session(struct tty_struct *tty); void tty_open_proc_set_tty(struct file *filp, struct tty_struct *tty); int tty_signal_session_leader(struct tty_struct *tty, int exit_session); diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 094e82a12d2988e22e91e41eed494fe1e54d593d..984e3098e631775e306de9003ba8146473c48ba6 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -948,7 +948,7 @@ void tty_write_unlock(struct tty_struct *tty) wake_up_interruptible_poll(&tty->write_wait, EPOLLOUT); } -int tty_write_lock(struct tty_struct *tty, int ndelay) +int tty_write_lock(struct tty_struct *tty, bool ndelay) { if (!mutex_trylock(&tty->atomic_write_lock)) { if (ndelay) @@ -1167,7 +1167,7 @@ int tty_send_xchar(struct tty_struct *tty, char ch) return 0; } - if (tty_write_lock(tty, 0) < 0) + if (tty_write_lock(tty, false) < 0) return -ERESTARTSYS; down_read(&tty->termios_rwsem); @@ -2470,22 +2470,25 @@ static int send_break(struct tty_struct *tty, unsigned int duration) return 0; if (tty->driver->flags & TTY_DRIVER_HARDWARE_BREAK) - retval = tty->ops->break_ctl(tty, duration); - else { - /* Do the work ourselves */ - if (tty_write_lock(tty, 0) < 0) - return -EINTR; - retval = tty->ops->break_ctl(tty, -1); - if (retval) - goto out; - if (!signal_pending(current)) - msleep_interruptible(duration); + return tty->ops->break_ctl(tty, duration); + + /* Do the work ourselves */ + if (tty_write_lock(tty, false) < 0) + return -EINTR; + + retval = tty->ops->break_ctl(tty, -1); + if (!retval) { + msleep_interruptible(duration); retval = tty->ops->break_ctl(tty, 0); -out: - tty_write_unlock(tty); - if (signal_pending(current)) - retval = -EINTR; + } else if (retval == -EOPNOTSUPP) { + /* some drivers can tell only dynamically */ + retval = 0; } + tty_write_unlock(tty); + + if (signal_pending(current)) + retval = -EINTR; + return retval; } diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c index 68b07250dcb607bf1819c8c6fd73f86807543050..12a30329abdb04ec274774e08620954f2a50151e 100644 --- a/drivers/tty/tty_ioctl.c +++ b/drivers/tty/tty_ioctl.c @@ -404,7 +404,7 @@ static int set_termios(struct tty_struct *tty, void __user *arg, int opt) if (retval < 0) return retval; - if (tty_write_lock(tty, 0) < 0) + if (tty_write_lock(tty, false) < 0) goto retry_write_wait; /* Racing writer? */ diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 7c54753c5439edab0696306df6e010a10121677f..b10b86e2c17e92ad484b95df1a8299ca872f46f1 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -599,8 +599,8 @@ static int vt_setactivate(struct vt_setactivate __user *sa) if (vsa.console == 0 || vsa.console > MAX_NR_CONSOLES) return -ENXIO; - vsa.console = array_index_nospec(vsa.console, MAX_NR_CONSOLES + 1); vsa.console--; + vsa.console = array_index_nospec(vsa.console, MAX_NR_CONSOLES); console_lock(); ret = vc_allocate(vsa.console); if (ret) { diff --git a/drivers/usb/cdns3/ep0.c b/drivers/usb/cdns3/ep0.c index 30d3516c7f9880428fd7e4c7714d663fb1907633..4241c513b9f62a1c1f151ef1b9b39da46633a2ea 100644 --- a/drivers/usb/cdns3/ep0.c +++ b/drivers/usb/cdns3/ep0.c @@ -364,7 +364,7 @@ static int cdns3_ep0_feature_handle_endpoint(struct cdns3_device *priv_dev, if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) return -EINVAL; - if (!(ctrl->wIndex & ~USB_DIR_IN)) + if (!(le16_to_cpu(ctrl->wIndex) & ~USB_DIR_IN)) return 0; index = cdns3_ep_addr_to_index(le16_to_cpu(ctrl->wIndex)); @@ -790,7 +790,7 @@ int cdns3_gadget_ep_set_wedge(struct usb_ep *ep) return 0; } -const struct usb_ep_ops cdns3_gadget_ep0_ops = { +static const struct usb_ep_ops cdns3_gadget_ep0_ops = { .enable = cdns3_gadget_ep0_enable, .disable = cdns3_gadget_ep0_disable, .alloc_request = cdns3_gadget_ep_alloc_request, diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c index 210c1d61508253d8d168036cf9b2c91e0e5e361b..8a1f0a636848bdeba0c55512b961a178a440fe13 100644 --- a/drivers/usb/cdns3/gadget.c +++ b/drivers/usb/cdns3/gadget.c @@ -1118,6 +1118,8 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, dma_addr_t trb_dma; u32 togle_pcs = 1; int sg_iter = 0; + int num_trb_req; + int trb_burst; int num_trb; int address; u32 control; @@ -1126,15 +1128,13 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, struct scatterlist *s = NULL; bool sg_supported = !!(request->num_mapped_sgs); + num_trb_req = sg_supported ? request->num_mapped_sgs : 1; + + /* ISO transfer require each SOF have a TD, each TD include some TRBs */ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) - num_trb = priv_ep->interval; + num_trb = priv_ep->interval * num_trb_req; else - num_trb = sg_supported ? request->num_mapped_sgs : 1; - - if (num_trb > priv_ep->free_trbs) { - priv_ep->flags |= EP_RING_FULL; - return -ENOBUFS; - } + num_trb = num_trb_req; priv_req = to_cdns3_request(request); address = priv_ep->endpoint.desc->bEndpointAddress; @@ -1183,14 +1183,31 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) | TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit); + + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { + /* + * ISO require LINK TRB must be first one of TD. + * Fill LINK TRBs for left trb space to simply software process logic. + */ + while (priv_ep->enqueue) { + *trb = *link_trb; + trace_cdns3_prepare_trb(priv_ep, trb); + + cdns3_ep_inc_enq(priv_ep); + trb = priv_ep->trb_pool + priv_ep->enqueue; + priv_req->trb = trb; + } + } + } + + if (num_trb > priv_ep->free_trbs) { + priv_ep->flags |= EP_RING_FULL; + return -ENOBUFS; } if (priv_dev->dev_ver <= DEV_VER_V2) togle_pcs = cdns3_wa1_update_guard(priv_ep, trb); - if (sg_supported) - s = request->sg; - /* set incorrect Cycle Bit for first trb*/ control = priv_ep->pcs ? 0 : TRB_CYCLE; trb->length = 0; @@ -1200,7 +1217,7 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, td_size = DIV_ROUND_UP(request->length, priv_ep->endpoint.maxpacket); if (priv_dev->gadget.speed == USB_SPEED_SUPER) - trb->length = TRB_TDL_SS_SIZE(td_size); + trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size)); else control |= TRB_TDL_HS_SIZE(td_size); } @@ -1208,6 +1225,9 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, do { u32 length; + if (!(sg_iter % num_trb_req) && sg_supported) + s = request->sg; + /* fill TRB */ control |= TRB_TYPE(TRB_NORMAL); if (sg_supported) { @@ -1222,7 +1242,36 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, total_tdl += DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket); - trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) | + trb_burst = priv_ep->trb_burst_size; + + /* + * Supposed DMA cross 4k bounder problem should be fixed at DEV_VER_V2, but still + * met problem when do ISO transfer if sg enabled. + * + * Data pattern likes below when sg enabled, package size is 1k and mult is 2 + * [UVC Header(8B) ] [data(3k - 8)] ... + * + * The received data at offset 0xd000 will get 0xc000 data, len 0x70. Error happen + * as below pattern: + * 0xd000: wrong + * 0xe000: wrong + * 0xf000: correct + * 0x10000: wrong + * 0x11000: wrong + * 0x12000: correct + * ... + * + * But it is still unclear about why error have not happen below 0xd000, it should + * cross 4k bounder. But anyway, the below code can fix this problem. + * + * To avoid DMA cross 4k bounder at ISO transfer, reduce burst len according to 16. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2) + if (ALIGN_DOWN(trb->buffer, SZ_4K) != + ALIGN_DOWN(trb->buffer + length, SZ_4K)) + trb_burst = 16; + + trb->length |= cpu_to_le32(TRB_BURST_LEN(trb_burst) | TRB_LEN(length)); pcs = priv_ep->pcs ? TRB_CYCLE : 0; @@ -1247,10 +1296,10 @@ static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, priv_req->trb->control = cpu_to_le32(control); if (sg_supported) { - trb->control |= TRB_ISP; + trb->control |= cpu_to_le32(TRB_ISP); /* Don't set chain bit for last TRB */ - if (sg_iter < num_trb - 1) - trb->control |= TRB_CHAIN; + if ((sg_iter % num_trb_req) < num_trb_req - 1) + trb->control |= cpu_to_le32(TRB_CHAIN); s = sg_next(s); } @@ -1507,6 +1556,12 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev, /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */ while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) { + + /* ISO ep_traddr may stop at LINK TRB */ + if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) && + priv_ep->type == USB_ENDPOINT_XFER_ISOC) + break; + trace_cdns3_complete_trb(priv_ep, trb); cdns3_ep_inc_deq(priv_ep); trb = priv_ep->trb_pool + priv_ep->dequeue; @@ -1539,6 +1594,10 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev, } if (request_handled) { + /* TRBs are duplicated by priv_ep->interval time for ISO IN */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir) + request->actual /= priv_ep->interval; + cdns3_gadget_giveback(priv_ep, priv_req, 0); request_handled = false; transfer_end = false; @@ -2034,11 +2093,10 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); struct cdns3_device *priv_dev = priv_ep->cdns3_dev; u32 bEndpointAddress = priv_ep->num | priv_ep->dir; - u32 max_packet_size = 0; - u8 maxburst = 0; + u32 max_packet_size = priv_ep->wMaxPacketSize; + u8 maxburst = priv_ep->bMaxBurst; u32 ep_cfg = 0; u8 buffering; - u8 mult = 0; int ret; buffering = priv_dev->ep_buf_size - 1; @@ -2060,8 +2118,7 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) break; default: ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC); - mult = priv_dev->ep_iso_burst - 1; - buffering = mult + 1; + buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1; } switch (priv_dev->gadget.speed) { @@ -2072,17 +2129,8 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) max_packet_size = is_iso_ep ? 1024 : 512; break; case USB_SPEED_SUPER: - /* It's limitation that driver assumes in driver. */ - mult = 0; - max_packet_size = 1024; - if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { - maxburst = priv_dev->ep_iso_burst - 1; - buffering = (mult + 1) * - (maxburst + 1); - - if (priv_ep->interval > 1) - buffering++; - } else { + if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) { + max_packet_size = 1024; maxburst = priv_dev->ep_buf_size - 1; } break; @@ -2111,7 +2159,6 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) if (priv_dev->dev_ver < DEV_VER_V2) priv_ep->trb_burst_size = 16; - mult = min_t(u8, mult, EP_CFG_MULT_MAX); buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX); maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX); @@ -2145,7 +2192,7 @@ int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) } ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) | - EP_CFG_MULT(mult) | + EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */ EP_CFG_BUFFERING(buffering) | EP_CFG_MAXBURST(maxburst); @@ -2235,6 +2282,13 @@ usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget, priv_ep->type = usb_endpoint_type(desc); priv_ep->flags |= EP_CLAIMED; priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; + priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc); + priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize); + priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK; + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) { + priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1; + priv_ep->bMaxBurst = comp_desc->bMaxBurst; + } spin_unlock_irqrestore(&priv_dev->lock, flags); return &priv_ep->endpoint; @@ -3001,23 +3055,43 @@ static int cdns3_gadget_udc_stop(struct usb_gadget *gadget) static int cdns3_gadget_check_config(struct usb_gadget *gadget) { struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + struct cdns3_endpoint *priv_ep; struct usb_ep *ep; int n_in = 0; + int iso = 0; + int out = 1; int total; + int n; list_for_each_entry(ep, &gadget->ep_list, ep_list) { - if (ep->claimed && (ep->address & USB_DIR_IN)) - n_in++; + priv_ep = ep_to_cdns3_ep(ep); + if (!(priv_ep->flags & EP_CLAIMED)) + continue; + + n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1); + if (ep->address & USB_DIR_IN) { + /* + * ISO transfer: DMA start move data when get ISO, only transfer + * data as min(TD size, iso). No benefit for allocate bigger + * internal memory than 'iso'. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) + iso += n; + else + n_in++; + } else { + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) + out = max_t(int, out, n); + } } /* 2KB are reserved for EP0, 1KB for out*/ - total = 2 + n_in + 1; + total = 2 + n_in + out + iso; if (total > priv_dev->onchip_buffers) return -ENOMEM; - priv_dev->ep_buf_size = priv_dev->ep_iso_burst = - (priv_dev->onchip_buffers - 2) / (n_in + 1); + priv_dev->ep_buf_size = (priv_dev->onchip_buffers - 2 - iso) / (n_in + out); return 0; } diff --git a/drivers/usb/cdns3/gadget.h b/drivers/usb/cdns3/gadget.h index 32825477edd3eae3bedcb18a4cc4a0f3c5959ceb..aeb2211228c13a523b0baf04f6d2a6d76b74b67d 100644 --- a/drivers/usb/cdns3/gadget.h +++ b/drivers/usb/cdns3/gadget.h @@ -1167,6 +1167,9 @@ struct cdns3_endpoint { u8 dir; u8 num; u8 type; + u8 mult; + u8 bMaxBurst; + u16 wMaxPacketSize; int interval; int free_trbs; diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c index 3d18599c5b9e4e8b99ddf4584e0288644e9d2f49..b8bb4cdadff8f3a72d773302bd231a2d2f6087da 100644 --- a/drivers/usb/chipidea/core.c +++ b/drivers/usb/chipidea/core.c @@ -516,6 +516,13 @@ static irqreturn_t ci_irq_handler(int irq, void *data) u32 otgsc = 0; if (ci->in_lpm) { + /* + * If we already have a wakeup irq pending there, + * let's just return to wait resume finished firstly. + */ + if (ci->wakeup_int) + return IRQ_HANDLED; + disable_irq_nosync(irq); ci->wakeup_int = true; pm_runtime_get(ci->dev); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 070b838c7da986ad1ac73180eafe10f8d219cc2b..4e4a71307d63c86c2f330492540a04c96ebd32c9 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -896,6 +896,9 @@ static int acm_tty_break_ctl(struct tty_struct *tty, int state) struct acm *acm = tty->driver_data; int retval; + if (!(acm->ctrl_caps & USB_CDC_CAP_BRK)) + return -EOPNOTSUPP; + retval = acm_send_break(acm, state ? 0xffff : 0); if (retval < 0) dev_dbg(&acm->control->dev, diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index 2f8a1225b6976bbd30ac22947b8e732012fd8030..1508e0f00dbc69c06da4f09e1dc486f175b31fac 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -291,6 +291,20 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, if (ifp->desc.bNumEndpoints >= num_ep) goto skip_to_next_endpoint_or_interface_descriptor; + /* Save a copy of the descriptor and use it instead of the original */ + endpoint = &ifp->endpoint[ifp->desc.bNumEndpoints]; + memcpy(&endpoint->desc, d, n); + d = &endpoint->desc; + + /* Clear the reserved bits in bEndpointAddress */ + i = d->bEndpointAddress & + (USB_ENDPOINT_DIR_MASK | USB_ENDPOINT_NUMBER_MASK); + if (i != d->bEndpointAddress) { + dev_notice(ddev, "config %d interface %d altsetting %d has an endpoint descriptor with address 0x%X, changing to 0x%X\n", + cfgno, inum, asnum, d->bEndpointAddress, i); + endpoint->desc.bEndpointAddress = i; + } + /* Check for duplicate endpoint addresses */ if (config_endpoint_is_duplicate(config, inum, asnum, d)) { dev_notice(ddev, "config %d interface %d altsetting %d has a duplicate endpoint with address 0x%X, skipping\n", @@ -308,10 +322,8 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, } } - endpoint = &ifp->endpoint[ifp->desc.bNumEndpoints]; + /* Accept this endpoint */ ++ifp->desc.bNumEndpoints; - - memcpy(&endpoint->desc, d, n); INIT_LIST_HEAD(&endpoint->urb_list); /* diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 214a8ff2d69c8522994af401bde03d4e43d1e019..26f9928b972c6515892bc5622feb35d9275f5860 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -275,48 +275,11 @@ int dwc3_core_soft_reset(struct dwc3 *dwc) /* * We're resetting only the device side because, if we're in host mode, * XHCI driver will reset the host block. If dwc3 was configured for - * host-only mode or current role is host, then we can return early. + * host-only mode, then we can return early. */ if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_HOST) return 0; - /* - * If the dr_mode is host and the dwc->current_dr_role is not the - * corresponding DWC3_GCTL_PRTCAP_HOST, then the dwc3_core_init_mode - * isn't executed yet. Ensure the phy is ready before the controller - * updates the GCTL.PRTCAPDIR or other settings by soft-resetting - * the phy. - * - * Note: GUSB3PIPECTL[n] and GUSB2PHYCFG[n] are port settings where n - * is port index. If this is a multiport host, then we need to reset - * all active ports. - */ - if (dwc->dr_mode == USB_DR_MODE_HOST) { - u32 usb3_port; - u32 usb2_port; - - usb3_port = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0)); - usb3_port |= DWC3_GUSB3PIPECTL_PHYSOFTRST; - dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), usb3_port); - - usb2_port = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); - usb2_port |= DWC3_GUSB2PHYCFG_PHYSOFTRST; - dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), usb2_port); - - /* Small delay for phy reset assertion */ - usleep_range(1000, 2000); - - usb3_port &= ~DWC3_GUSB3PIPECTL_PHYSOFTRST; - dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), usb3_port); - - usb2_port &= ~DWC3_GUSB2PHYCFG_PHYSOFTRST; - dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), usb2_port); - - /* Wait for clock synchronization */ - msleep(50); - return 0; - } - reg = dwc3_readl(dwc->regs, DWC3_DCTL); reg |= DWC3_DCTL_CSFTRST; reg &= ~DWC3_DCTL_RUN_STOP; diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 3cd294264372555ac10318e558dbdbf68c15438f..14bdef97090b2a11e7d7841cbad33872a8786eaa 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -236,7 +236,10 @@ static void dwc3_ep0_stall_and_restart(struct dwc3 *dwc) struct dwc3_request *req; req = next_request(&dep->pending_list); - dwc3_gadget_giveback(dep, req, -ECONNRESET); + if (!dwc->connected) + dwc3_gadget_giveback(dep, req, -ESHUTDOWN); + else + dwc3_gadget_giveback(dep, req, -ECONNRESET); } dwc->ep0state = EP0_SETUP_PHASE; diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index d51ea1c052f244f65148dca42efb556367f0b817..6bb69a4e64704f106112edfc6f4a969ef7eb092f 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -104,9 +104,12 @@ static int usb_string_copy(const char *s, char **s_copy) int ret; char *str; char *copy = *s_copy; + ret = strlen(s); if (ret > USB_MAX_STRING_LEN) return -EOVERFLOW; + if (ret < 1) + return -EINVAL; if (copy) { str = copy; diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index 73e13e7c2b4658be17326ae5821fadab737f8395..0a859891bf35f669e54cc1c3a04a389381354de8 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -1132,8 +1132,10 @@ int ohci_resume(struct usb_hcd *hcd, bool hibernated) set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags); /* Make sure resume from hibernation re-enumerates everything */ - if (hibernated) + if (hibernated) { + ohci_writel(ohci, OHCI_INTR_MIE, &ohci->regs->intrdisable); ohci_usb_reset(ohci); + } /* See if the controller is already running or has been reset */ ohci->hc_control = ohci_readl(ohci, &ohci->regs->control); diff --git a/drivers/usb/host/ohci-hub.c b/drivers/usb/host/ohci-hub.c index 44504c1751e0450e309abb1338cc97df0556f26d..8651230245684794afe1ce9d4493cd911da60969 100644 --- a/drivers/usb/host/ohci-hub.c +++ b/drivers/usb/host/ohci-hub.c @@ -91,6 +91,9 @@ __acquires(ohci->lock) update_done_list(ohci); ohci_work(ohci); + /* All ED unlinks should be finished, no need for SOF interrupts */ + ohci_writel(ohci, OHCI_INTR_SF, &ohci->regs->intrdisable); + /* * Some controllers don't handle "global" suspend properly if * there are unsuspended ports. For these controllers, put all diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index ef08d68b9714924513a3e5b47d467f4793e18f73..2975e592e231d06d606bf413f6c970dedf397aff 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -948,6 +948,10 @@ static void quirk_usb_disable_ehci(struct pci_dev *pdev) * booting from USB disk or using a usb keyboard */ hcc_params = readl(base + EHCI_HCC_PARAMS); + if (pdev->vendor == PCI_VENDOR_ID_LOONGSON && + pdev->device == PCI_DEVICE_ID_LOONGSON_EHCI) + hcc_params &= ~(0xffL << 8); + offset = (hcc_params >> 8) & 0xff; while (offset && --count) { pci_read_config_dword(pdev, offset, &cap); diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 53f8327873501aec69bf883b5a42a927f9a85562..311e96b05c53439d508b502132c0fa2c1c256604 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -274,11 +274,13 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == 0x0014) { xhci->quirks |= XHCI_TRUST_TX_LENGTH; xhci->quirks |= XHCI_ZERO_64B_REGS; + xhci->quirks |= XHCI_LWP_QUIRK; } if (pdev->vendor == PCI_VENDOR_ID_RENESAS && pdev->device == 0x0015) { xhci->quirks |= XHCI_RESET_ON_RESUME; xhci->quirks |= XHCI_ZERO_64B_REGS; + xhci->quirks |= XHCI_LWP_QUIRK; } if (pdev->vendor == PCI_VENDOR_ID_VIA) xhci->quirks |= XHCI_RESET_ON_RESUME; @@ -544,6 +546,10 @@ static void xhci_pci_remove(struct pci_dev *dev) if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) pci_set_power_state(dev, PCI_D3hot); + /* Workaround for decreasing power consumption after S5 */ + if (xhci->quirks & XHCI_LWP_QUIRK) + pci_set_power_state(dev, PCI_D3hot); + usb_hcd_pci_remove(dev); } @@ -689,6 +695,10 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd) /* Yet another workaround for spurious wakeups at shutdown with HSW */ if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) pci_set_power_state(pdev, PCI_D3hot); + + /* Workaround for decreasing power consumption after S5 */ + if (xhci->quirks & XHCI_LWP_QUIRK) + pci_set_power_state(pdev, PCI_D3hot); } #endif /* CONFIG_PM */ diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 4fa387e447f08233f952774dae72a10e3a4aa93e..89044e610c7e1215196784cbf17aefbd55eebae7 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -2482,16 +2482,17 @@ static int handle_tx_event(struct xhci_hcd *xhci, else xhci_handle_halted_endpoint(xhci, ep, 0, NULL, EP_SOFT_RESET); - goto cleanup; + break; case COMP_RING_UNDERRUN: case COMP_RING_OVERRUN: case COMP_STOPPED_LENGTH_INVALID: - goto cleanup; + break; default: xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n", slot_id, ep_index); goto err_out; } + return 0; } /* Count current td numbers if ep->skip is set */ diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 5a8443f6ed703e8a36149b10c0a2a9445cda343b..c6e6d73fa6f66a98ad3bbae753fe0ff7f6ee218b 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1899,6 +1899,7 @@ struct xhci_hcd { #define XHCI_RESET_TO_DEFAULT BIT_ULL(44) #define XHCI_ZHAOXIN_TRB_FETCH BIT_ULL(45) #define XHCI_ZHAOXIN_HOST BIT_ULL(46) +#define XHCI_LWP_QUIRK BIT_ULL(47) unsigned int num_active_eps; unsigned int limit_active_eps; diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c index 094e812e9e69223d3c7a866bb1f416ef73a9aeac..35483217b1f6cfc53ff01b67460e5efbd486da4e 100644 --- a/drivers/usb/mon/mon_bin.c +++ b/drivers/usb/mon/mon_bin.c @@ -1247,14 +1247,19 @@ static vm_fault_t mon_bin_vma_fault(struct vm_fault *vmf) struct mon_reader_bin *rp = vmf->vma->vm_private_data; unsigned long offset, chunk_idx; struct page *pageptr; + unsigned long flags; + spin_lock_irqsave(&rp->b_lock, flags); offset = vmf->pgoff << PAGE_SHIFT; - if (offset >= rp->b_size) + if (offset >= rp->b_size) { + spin_unlock_irqrestore(&rp->b_lock, flags); return VM_FAULT_SIGBUS; + } chunk_idx = offset / CHUNK_SIZE; pageptr = rp->b_vec[chunk_idx].pg; get_page(pageptr); vmf->page = pageptr; + spin_unlock_irqrestore(&rp->b_lock, flags); return 0; } diff --git a/drivers/usb/phy/phy-mxs-usb.c b/drivers/usb/phy/phy-mxs-usb.c index 70e23334b27f97371160e7a190c3651c9397d6c3..e3cddcac3252ce4f1f38dff0f6b53643a419d876 100644 --- a/drivers/usb/phy/phy-mxs-usb.c +++ b/drivers/usb/phy/phy-mxs-usb.c @@ -388,8 +388,7 @@ static void __mxs_phy_disconnect_line(struct mxs_phy *mxs_phy, bool disconnect) static bool mxs_phy_is_otg_host(struct mxs_phy *mxs_phy) { - return IS_ENABLED(CONFIG_USB_OTG) && - mxs_phy->phy.last_event == USB_EVENT_ID; + return mxs_phy->phy.last_event == USB_EVENT_ID; } static void mxs_phy_disconnect_line(struct mxs_phy *mxs_phy, bool on) diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 18b35e8173614c282d0ab8a7e48913ada397b08b..7fa95e7012446a314c86b2be7702d53897aea169 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -190,11 +190,13 @@ static void typec_altmode_put_partner(struct altmode *altmode) { struct altmode *partner = altmode->partner; struct typec_altmode *adev; + struct typec_altmode *partner_adev; if (!partner) return; adev = &altmode->adev; + partner_adev = &partner->adev; if (is_typec_plug(adev->dev.parent)) { struct typec_plug *plug = to_typec_plug(adev->dev.parent); @@ -203,7 +205,7 @@ static void typec_altmode_put_partner(struct altmode *altmode) } else { partner->partner = NULL; } - put_device(&adev->dev); + put_device(&partner_adev->dev); } /** diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index e44bf736e2b222e42e0727136ce5bdd43b319236..3a625e39601d47509ba33aa408189e3da45ccd20 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -22,7 +22,7 @@ config VFIO_VIRQFD menuconfig VFIO tristate "VFIO Non-Privileged userspace driver framework" select IOMMU_API - select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64) + select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64 || LOONGARCH) help VFIO provides a framework for secure userspace device drivers. See Documentation/driver-api/vfio.rst for more details. diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c index a591d291b231af8531174bd7dd8057d767ab13f1..0708e214c5a338baa4160b47befa5d966952dc6b 100644 --- a/drivers/video/fbdev/core/fb_defio.c +++ b/drivers/video/fbdev/core/fb_defio.c @@ -78,11 +78,7 @@ int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasy return 0; inode_lock(inode); - /* Kill off the delayed work */ - cancel_delayed_work_sync(&info->deferred_work); - - /* Run it immediately */ - schedule_delayed_work(&info->deferred_work, 0); + flush_delayed_work(&info->deferred_work); inode_unlock(inode); return 0; diff --git a/drivers/watchdog/bcm2835_wdt.c b/drivers/watchdog/bcm2835_wdt.c index dec6ca019beaaaaebdc52b4a88459a8327f9d2b8..3a8dec05b59118f7ce10bf95b690ba687e09551d 100644 --- a/drivers/watchdog/bcm2835_wdt.c +++ b/drivers/watchdog/bcm2835_wdt.c @@ -42,6 +42,7 @@ #define SECS_TO_WDOG_TICKS(x) ((x) << 16) #define WDOG_TICKS_TO_SECS(x) ((x) >> 16) +#define WDOG_TICKS_TO_MSECS(x) ((x) * 1000 >> 16) struct bcm2835_wdt { void __iomem *base; @@ -140,7 +141,7 @@ static struct watchdog_device bcm2835_wdt_wdd = { .info = &bcm2835_wdt_info, .ops = &bcm2835_wdt_ops, .min_timeout = 1, - .max_timeout = WDOG_TICKS_TO_SECS(PM_WDOG_TIME_SET), + .max_hw_heartbeat_ms = WDOG_TICKS_TO_MSECS(PM_WDOG_TIME_SET), .timeout = WDOG_TICKS_TO_SECS(PM_WDOG_TIME_SET), }; diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c index 7d34bcf1c45b4eb012f3fd1f42d832fac7adc672..53573c3ddd1a69b447ababab3be0dcb52bb2a108 100644 --- a/drivers/watchdog/hpwdt.c +++ b/drivers/watchdog/hpwdt.c @@ -174,7 +174,7 @@ static int hpwdt_pretimeout(unsigned int ulReason, struct pt_regs *regs) "3. OA Forward Progress Log\n" "4. iLO Event Log"; - if (ilo5 && ulReason == NMI_UNKNOWN && !mynmi) + if (ulReason == NMI_UNKNOWN && !mynmi) return NMI_DONE; if (ilo5 && !pretimeout && !mynmi) diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 46c2a4bd9ebe9871b997aeef23a2978d432ff60d..daa00f3c5a6af96f726c7413b3aaa7c522457262 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -70,6 +70,11 @@ static int rti_wdt_start(struct watchdog_device *wdd) { u32 timer_margin; struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd); + int ret; + + ret = pm_runtime_resume_and_get(wdd->parent); + if (ret) + return ret; /* set timeout period */ timer_margin = (u64)wdd->timeout * wdt->freq; @@ -296,6 +301,9 @@ static int rti_wdt_probe(struct platform_device *pdev) if (last_ping) watchdog_set_last_hw_keepalive(wdd, last_ping); + if (!watchdog_hw_running(wdd)) + pm_runtime_put_sync(&pdev->dev); + return 0; err_iomap: @@ -310,7 +318,10 @@ static int rti_wdt_remove(struct platform_device *pdev) struct rti_wdt_device *wdt = platform_get_drvdata(pdev); watchdog_unregister_device(&wdt->wdd); - pm_runtime_put(&pdev->dev); + + if (!pm_runtime_suspended(&pdev->dev)) + pm_runtime_put(&pdev->dev); + pm_runtime_disable(&pdev->dev); return 0; diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index f37255cd75fdf4d6e785cc4a72f522e45c345426..24d0470af81cab55b5a9ba723eb2a704c7da9bd4 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -1028,6 +1028,7 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) /* Fill in the data structures */ cdev_init(&wd_data->cdev, &watchdog_fops); + wd_data->cdev.owner = wdd->ops->owner; /* Add the device */ err = cdev_device_add(&wd_data->cdev, &wd_data->dev); @@ -1042,8 +1043,6 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) return err; } - wd_data->cdev.owner = wdd->ops->owner; - /* Record time of most recent heartbeat as 'just before now'. */ wd_data->last_hw_keepalive = ktime_sub(ktime_get(), 1); watchdog_set_open_deadline(wd_data); diff --git a/fs/Kconfig b/fs/Kconfig index 3b4a3f49188aab0799e02d36c95c0e5c1d05c916..82cda72913941e4eb5e5e33b21ba890e19c069f4 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -118,6 +118,8 @@ source "fs/crypto/Kconfig" source "fs/code_sign/Kconfig" +source "fs/dec/Kconfig" + source "fs/verity/Kconfig" source "fs/notify/Kconfig" diff --git a/fs/Makefile b/fs/Makefile index 5036d569163229243ed1e5cf45b82117e34523bd..0185494c9a0c6e17acfb9e49ccc9447d75f44656 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_AIO) += aio.o obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ obj-$(CONFIG_SECURITY_CODE_SIGN) += code_sign/ +obj-$(CONFIG_SECURITY_DEC) += dec/ obj-$(CONFIG_FS_VERITY) += verity/ obj-$(CONFIG_FILE_LOCKING) += locks.o obj-$(CONFIG_BINFMT_AOUT) += binfmt_aout.o diff --git a/fs/afs/callback.c b/fs/afs/callback.c index 7d9b23d981bf1aaac48478b3843e39a5c7f1f9bc..229308c7f7449bbe481e79b07d7fa7369ce2d946 100644 --- a/fs/afs/callback.c +++ b/fs/afs/callback.c @@ -70,13 +70,14 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, { struct afs_volume *volume = NULL; struct rb_node *p; - int seq = 0; + int seq = 1; do { /* Unfortunately, rbtree walking doesn't give reliable results * under just the RCU read lock, so we have to check for * changes. */ + seq++; /* 2 on the 1st/lockless path, otherwise odd */ read_seqbegin_or_lock(&cell->volume_lock, &seq); p = rcu_dereference_raw(cell->volumes.rb_node); diff --git a/fs/afs/dir.c b/fs/afs/dir.c index a59d6293a32b2f141073d65222c64a6cf5fadb45..0b927736ca72887f0a8842983eb73522770bcdbd 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -418,6 +418,14 @@ static int afs_dir_iterate_block(struct afs_vnode *dvnode, continue; } + /* Don't expose silly rename entries to userspace. */ + if (nlen > 6 && + dire->u.name[0] == '.' && + ctx->actor != afs_lookup_filldir && + ctx->actor != afs_lookup_one_filldir && + memcmp(dire->u.name, ".__afs", 6) == 0) + continue; + /* found the next entry */ if (!dir_emit(ctx, dire->u.name, nlen, ntohl(dire->u.vnode), diff --git a/fs/afs/server.c b/fs/afs/server.c index 684a2b02b9ff70ae056aab6a3807f624bb26c47a..733e3c470f7e31e79eb58c9d353c2438eec35e54 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -27,7 +27,7 @@ struct afs_server *afs_find_server(struct afs_net *net, const struct afs_addr_list *alist; struct afs_server *server = NULL; unsigned int i; - int seq = 0, diff; + int seq = 1, diff; rcu_read_lock(); @@ -35,6 +35,7 @@ struct afs_server *afs_find_server(struct afs_net *net, if (server) afs_unuse_server_notime(net, server, afs_server_trace_put_find_rsq); server = NULL; + seq++; /* 2 on the 1st/lockless path, otherwise odd */ read_seqbegin_or_lock(&net->fs_addr_lock, &seq); if (srx->transport.family == AF_INET6) { @@ -90,7 +91,7 @@ struct afs_server *afs_find_server_by_uuid(struct afs_net *net, const uuid_t *uu { struct afs_server *server = NULL; struct rb_node *p; - int diff, seq = 0; + int diff, seq = 1; _enter("%pU", uuid); @@ -102,7 +103,7 @@ struct afs_server *afs_find_server_by_uuid(struct afs_net *net, const uuid_t *uu if (server) afs_unuse_server(net, server, afs_server_trace_put_uuid_rsq); server = NULL; - + seq++; /* 2 on the 1st/lockless path, otherwise odd */ read_seqbegin_or_lock(&net->fs_lock, &seq); p = net->fs_servers.rb_node; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 8f62e171053ba86bd13d0cd72315c055556527cb..3ba43a40032cda05b83f1810581579b705297c08 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1202,7 +1202,8 @@ static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, u64 bytes_left, end; u64 aligned_start = ALIGN(start, 1 << 9); - if (WARN_ON(start != aligned_start)) { + /* Adjust the range to be aligned to 512B sectors if necessary. */ + if (start != aligned_start) { len -= aligned_start - start; len = round_down(len, 1 << 9); start = aligned_start; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index c900a39666e38bd75c9829c3e595c9101e65c138..250b6064876de78a9fd341c79d9af6d372339e10 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4007,7 +4007,8 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) struct btrfs_block_rsv block_rsv; u64 root_flags; int ret; - int err; + + down_write(&fs_info->subvol_sem); /* * Don't allow to delete a subvolume with send in progress. This is @@ -4020,25 +4021,25 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) btrfs_warn(fs_info, "attempt to delete subvolume %llu during send", dest->root_key.objectid); - return -EPERM; + ret = -EPERM; + goto out_up_write; } if (atomic_read(&dest->nr_swapfiles)) { spin_unlock(&dest->root_item_lock); btrfs_warn(fs_info, "attempt to delete subvolume %llu with active swapfile", root->root_key.objectid); - return -EPERM; + ret = -EPERM; + goto out_up_write; } root_flags = btrfs_root_flags(&dest->root_item); btrfs_set_root_flags(&dest->root_item, root_flags | BTRFS_ROOT_SUBVOL_DEAD); spin_unlock(&dest->root_item_lock); - down_write(&fs_info->subvol_sem); - - err = may_destroy_subvol(dest); - if (err) - goto out_up_write; + ret = may_destroy_subvol(dest); + if (ret) + goto out_undead; btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); /* @@ -4046,13 +4047,13 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) * two for dir entries, * two for root ref/backref. */ - err = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); - if (err) - goto out_up_write; + ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); + if (ret) + goto out_undead; trans = btrfs_start_transaction(root, 0); if (IS_ERR(trans)) { - err = PTR_ERR(trans); + ret = PTR_ERR(trans); goto out_release; } trans->block_rsv = &block_rsv; @@ -4062,7 +4063,6 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) ret = btrfs_unlink_subvol(trans, dir, dentry); if (ret) { - err = ret; btrfs_abort_transaction(trans, ret); goto out_end_trans; } @@ -4080,7 +4080,6 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) dest->root_key.objectid); if (ret) { btrfs_abort_transaction(trans, ret); - err = ret; goto out_end_trans; } } @@ -4090,7 +4089,6 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) dest->root_key.objectid); if (ret && ret != -ENOENT) { btrfs_abort_transaction(trans, ret); - err = ret; goto out_end_trans; } if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { @@ -4100,7 +4098,6 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) dest->root_key.objectid); if (ret && ret != -ENOENT) { btrfs_abort_transaction(trans, ret); - err = ret; goto out_end_trans; } } @@ -4111,20 +4108,20 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) trans->block_rsv = NULL; trans->bytes_reserved = 0; ret = btrfs_end_transaction(trans); - if (ret && !err) - err = ret; inode->i_flags |= S_DEAD; out_release: btrfs_subvolume_release_metadata(root, &block_rsv); -out_up_write: - up_write(&fs_info->subvol_sem); - if (err) { +out_undead: + if (ret) { spin_lock(&dest->root_item_lock); root_flags = btrfs_root_flags(&dest->root_item); btrfs_set_root_flags(&dest->root_item, root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); spin_unlock(&dest->root_item_lock); - } else { + } +out_up_write: + up_write(&fs_info->subvol_sem); + if (!ret) { d_invalidate(dentry); btrfs_prune_dentries(dest); ASSERT(dest->send_in_progress == 0); @@ -4136,7 +4133,7 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) } } - return err; + return ret; } static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index f06824bea4686c4d08f7dc6306547a87315f9b2c..4cda79a60bccdbbbeca78c3046d306d9a03b49b3 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -798,6 +798,9 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, struct btrfs_trans_handle *trans; int ret; + if (btrfs_root_refs(&root->root_item) == 0) + return -ENOENT; + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return -EINVAL; @@ -3190,6 +3193,10 @@ static int btrfs_ioctl_defrag(struct file *file, void __user *argp) kfree(range); goto out; } + if (range->flags & ~BTRFS_DEFRAG_RANGE_FLAGS_SUPP) { + ret = -EOPNOTSUPP; + goto out; + } /* compression requires us to start the IO */ if ((range->flags & BTRFS_DEFRAG_RANGE_COMPRESS)) { range->flags |= BTRFS_DEFRAG_RANGE_START_IO; diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c index bd3bb94cc56bd204ee6560ddbf4c90262556e57d..c3711598a9be55265ce6429b0f6adff5e04334e2 100644 --- a/fs/btrfs/ref-verify.c +++ b/fs/btrfs/ref-verify.c @@ -899,8 +899,10 @@ int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, out_unlock: spin_unlock(&fs_info->ref_verify_lock); out: - if (ret) + if (ret) { + btrfs_free_ref_cache(fs_info); btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); + } return ret; } @@ -1029,8 +1031,8 @@ int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) } } if (ret) { - btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); btrfs_free_ref_cache(fs_info); + btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); } btrfs_free_path(path); return ret; diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 32f1b15b25dcc34b027b3f4883e36b12e4793e4c..c0eda3816f6853ee0c8d0eba23994797d3f6c573 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1334,7 +1334,7 @@ static int check_extent_item(struct extent_buffer *leaf, if (ptr + btrfs_extent_inline_ref_size(inline_type) > end) { extent_err(leaf, slot, "inline ref item overflows extent item, ptr %lu iref size %u end %lu", - ptr, inline_type, end); + ptr, btrfs_extent_inline_ref_size(inline_type), end); return -EUCLEAN; } diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index b98bba887f84b0ab0a4145eaaf21de7de6332bb7..660e00eb42060ac4798aa22d61bbf979c2b1cce4 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -117,7 +117,7 @@ static __u32 get_neg_ctxt_len(struct smb2_sync_hdr *hdr, __u32 len, } else if (nc_offset + 1 == non_ctxlen) { cifs_dbg(FYI, "no SPNEGO security blob in negprot rsp\n"); size_of_pad_before_neg_ctxts = 0; - } else if (non_ctxlen == SMB311_NEGPROT_BASE_SIZE) + } else if (non_ctxlen == SMB311_NEGPROT_BASE_SIZE + 1) /* has padding, but no SPNEGO blob */ size_of_pad_before_neg_ctxts = nc_offset - non_ctxlen + 1; else diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 26edaeb4245d8c315aea4a06528302405e3881cf..84850a55c8b7e70ac875c6a9e152d2d613d6b69a 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -5561,7 +5561,7 @@ struct smb_version_values smb20_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5583,7 +5583,7 @@ struct smb_version_values smb21_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5604,7 +5604,7 @@ struct smb_version_values smb3any_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5625,7 +5625,7 @@ struct smb_version_values smbdefault_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5646,7 +5646,7 @@ struct smb_version_values smb30_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5667,7 +5667,7 @@ struct smb_version_values smb302_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, @@ -5688,7 +5688,7 @@ struct smb_version_values smb311_values = { .header_size = sizeof(struct smb2_sync_hdr), .header_preamble_size = 0, .max_header_size = MAX_SMB2_HDR_SIZE, - .read_rsp_size = sizeof(struct smb2_read_rsp) - 1, + .read_rsp_size = sizeof(struct smb2_read_rsp), .lock_cmd = SMB2_LOCK, .cap_unix = 0, .cap_nt_find = SMB2_NT_FIND, diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 76679dc4e63288643580079de5fe7eb65a141145..4aec01841f0f26ceb3e872169578a0b5ede4c530 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -1261,7 +1261,7 @@ SMB2_sess_sendreceive(struct SMB2_sess_data *sess_data) /* Testing shows that buffer offset must be at location of Buffer[0] */ req->SecurityBufferOffset = - cpu_to_le16(sizeof(struct smb2_sess_setup_req) - 1 /* pad */); + cpu_to_le16(sizeof(struct smb2_sess_setup_req)); req->SecurityBufferLength = cpu_to_le16(sess_data->iov[1].iov_len); memset(&rqst, 0, sizeof(struct smb_rqst)); @@ -1760,8 +1760,7 @@ SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, iov[0].iov_len = total_len - 1; /* Testing shows that buffer offset must be at location of Buffer[0] */ - req->PathOffset = cpu_to_le16(sizeof(struct smb2_tree_connect_req) - - 1 /* pad */); + req->PathOffset = cpu_to_le16(sizeof(struct smb2_tree_connect_req)); req->PathLength = cpu_to_le16(unc_path_len - 2); iov[1].iov_base = unc_path; iov[1].iov_len = unc_path_len; @@ -4676,7 +4675,7 @@ int SMB2_query_directory_init(const unsigned int xid, memcpy(bufptr, &asteriks, len); req->FileNameOffset = - cpu_to_le16(sizeof(struct smb2_query_directory_req) - 1); + cpu_to_le16(sizeof(struct smb2_query_directory_req)); req->FileNameLength = cpu_to_le16(len); /* * BB could be 30 bytes or so longer if we used SMB2 specific @@ -4873,7 +4872,7 @@ SMB2_set_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, req->AdditionalInformation = cpu_to_le32(additional_info); req->BufferOffset = - cpu_to_le16(sizeof(struct smb2_set_info_req) - 1); + cpu_to_le16(sizeof(struct smb2_set_info_req)); req->BufferLength = cpu_to_le32(*size); memcpy(req->Buffer, *data, *size); @@ -5105,9 +5104,9 @@ build_qfs_info_req(struct kvec *iov, struct cifs_tcon *tcon, req->VolatileFileId = volatile_fid; /* 1 for pad */ req->InputBufferOffset = - cpu_to_le16(sizeof(struct smb2_query_info_req) - 1); + cpu_to_le16(sizeof(struct smb2_query_info_req)); req->OutputBufferLength = cpu_to_le32( - outbuf_len + sizeof(struct smb2_query_info_rsp) - 1); + outbuf_len + sizeof(struct smb2_query_info_rsp)); iov->iov_base = (char *)req; iov->iov_len = total_len; diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index 89a732b31390ea505ea1a72d3f13f9b5fd5cdb29..eaa873175318a0cd569fd8233354b5c88e230909 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -220,7 +220,7 @@ struct smb2_err_rsp { __le16 StructureSize; __le16 Reserved; /* MBZ */ __le32 ByteCount; /* even if zero, at least one byte follows */ - __u8 ErrorData[1]; /* variable length */ + __u8 ErrorData[]; /* variable length */ } __packed; #define SYMLINK_ERROR_TAG 0x4c4d5953 @@ -464,7 +464,7 @@ struct smb2_negotiate_rsp { __le16 SecurityBufferOffset; __le16 SecurityBufferLength; __le32 NegotiateContextOffset; /* Pre:SMB3.1.1 was reserved/ignored */ - __u8 Buffer[1]; /* variable length GSS security buffer */ + __u8 Buffer[]; /* variable length GSS security buffer */ } __packed; /* Flags */ @@ -481,7 +481,7 @@ struct smb2_sess_setup_req { __le16 SecurityBufferOffset; __le16 SecurityBufferLength; __u64 PreviousSessionId; - __u8 Buffer[1]; /* variable length GSS security buffer */ + __u8 Buffer[]; /* variable length GSS security buffer */ } __packed; /* Currently defined SessionFlags */ @@ -494,7 +494,7 @@ struct smb2_sess_setup_rsp { __le16 SessionFlags; __le16 SecurityBufferOffset; __le16 SecurityBufferLength; - __u8 Buffer[1]; /* variable length GSS security buffer */ + __u8 Buffer[]; /* variable length GSS security buffer */ } __packed; struct smb2_logoff_req { @@ -520,7 +520,7 @@ struct smb2_tree_connect_req { __le16 Flags; /* Reserved MBZ for dialects prior to SMB3.1.1 */ __le16 PathOffset; __le16 PathLength; - __u8 Buffer[1]; /* variable length */ + __u8 Buffer[]; /* variable length */ } __packed; /* See MS-SMB2 section 2.2.9.2 */ @@ -828,7 +828,7 @@ struct smb2_create_rsp { __u64 VolatileFileId; /* opaque endianness */ __le32 CreateContextsOffset; __le32 CreateContextsLength; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; struct create_context { @@ -1289,7 +1289,7 @@ struct smb2_read_plain_req { __le32 RemainingBytes; __le16 ReadChannelInfoOffset; __le16 ReadChannelInfoLength; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; /* Read flags */ @@ -1304,7 +1304,7 @@ struct smb2_read_rsp { __le32 DataLength; __le32 DataRemaining; __u32 Flags; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; /* For write request Flags field below the following flags are defined: */ @@ -1324,7 +1324,7 @@ struct smb2_write_req { __le16 WriteChannelInfoOffset; __le16 WriteChannelInfoLength; __le32 Flags; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; struct smb2_write_rsp { @@ -1335,7 +1335,7 @@ struct smb2_write_rsp { __le32 DataLength; __le32 DataRemaining; __u32 Reserved2; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; /* notify flags */ @@ -1371,7 +1371,7 @@ struct smb2_change_notify_rsp { __le16 StructureSize; /* Must be 9 */ __le16 OutputBufferOffset; __le32 OutputBufferLength; - __u8 Buffer[1]; /* array of file notify structs */ + __u8 Buffer[]; /* array of file notify structs */ } __packed; #define SMB2_LOCKFLAG_SHARED_LOCK 0x0001 @@ -1394,7 +1394,10 @@ struct smb2_lock_req { __u64 PersistentFileId; /* opaque endianness */ __u64 VolatileFileId; /* opaque endianness */ /* Followed by at least one */ - struct smb2_lock_element locks[1]; + union { + struct smb2_lock_element lock; + DECLARE_FLEX_ARRAY(struct smb2_lock_element, locks); + }; } __packed; struct smb2_lock_rsp { @@ -1434,7 +1437,7 @@ struct smb2_query_directory_req { __le16 FileNameOffset; __le16 FileNameLength; __le32 OutputBufferLength; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; struct smb2_query_directory_rsp { @@ -1442,7 +1445,7 @@ struct smb2_query_directory_rsp { __le16 StructureSize; /* Must be 9 */ __le16 OutputBufferOffset; __le32 OutputBufferLength; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; /* Possible InfoType values */ @@ -1483,7 +1486,7 @@ struct smb2_query_info_req { __le32 Flags; __u64 PersistentFileId; /* opaque endianness */ __u64 VolatileFileId; /* opaque endianness */ - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; struct smb2_query_info_rsp { @@ -1491,7 +1494,7 @@ struct smb2_query_info_rsp { __le16 StructureSize; /* Must be 9 */ __le16 OutputBufferOffset; __le32 OutputBufferLength; - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; /* @@ -1514,7 +1517,7 @@ struct smb2_set_info_req { __le32 AdditionalInformation; __u64 PersistentFileId; /* opaque endianness */ __u64 VolatileFileId; /* opaque endianness */ - __u8 Buffer[1]; + __u8 Buffer[]; } __packed; struct smb2_set_info_rsp { @@ -1716,7 +1719,10 @@ struct smb2_file_all_info { /* data block encoding of response to level 18 */ __le32 Mode; __le32 AlignmentRequirement; __le32 FileNameLength; - char FileName[1]; + union { + char __pad; /* Legacy structure padding */ + DECLARE_FLEX_ARRAY(char, FileName); + }; } __packed; /* level 18 Query */ struct smb2_file_eof_info { /* encoding of request for level 10 */ diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index e23752d9a79f3d255345177db1b66950ae8e6607..c867a0d62f360b69a9d912731be9da76c487ecec 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -76,6 +76,14 @@ static struct inode *__ecryptfs_get_inode(struct inode *lower_inode, if (lower_inode->i_sb != ecryptfs_superblock_to_lower(sb)) return ERR_PTR(-EXDEV); + + /* Reject dealing with casefold directories. */ + if (IS_CASEFOLDED(lower_inode)) { + pr_err_ratelimited("%s: Can't handle casefolded directory.\n", + __func__); + return ERR_PTR(-EREMOTE); + } + if (!igrab(lower_inode)) return ERR_PTR(-ESTALE); inode = iget5_locked(sb, (unsigned long)lower_inode, diff --git a/fs/exec.c b/fs/exec.c index eedc61f186776d8a8b16aad2d507168ba3946740..3a65f7846d81af0f5ca19516e89edf09d6e3ecfb 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1401,6 +1401,9 @@ int begin_new_exec(struct linux_binprm * bprm) out_unlock: up_write(&me->signal->exec_update_lock); + if (!bprm->cred) + mutex_unlock(&me->signal->cred_guard_mutex); + out: return retval; } diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 931b200895710ff5153fc504ad399e7176b5ad3d..d66ba6f6a8115de975bbae9dda545ad96635ca5d 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -5918,11 +5918,16 @@ __acquires(bitlock) static ext4_grpblk_t ext4_last_grp_cluster(struct super_block *sb, ext4_group_t grp) { - if (grp < ext4_get_groups_count(sb)) - return EXT4_CLUSTERS_PER_GROUP(sb) - 1; - return (ext4_blocks_count(EXT4_SB(sb)->s_es) - - ext4_group_first_block_no(sb, grp) - 1) >> - EXT4_CLUSTER_BITS(sb); + unsigned long nr_clusters_in_group; + + if (grp < (ext4_get_groups_count(sb) - 1)) + nr_clusters_in_group = EXT4_CLUSTERS_PER_GROUP(sb); + else + nr_clusters_in_group = (ext4_blocks_count(EXT4_SB(sb)->s_es) - + ext4_group_first_block_no(sb, grp)) + >> EXT4_CLUSTER_BITS(sb); + + return nr_clusters_in_group - 1; } static bool ext4_trim_interrupted(void) @@ -5934,13 +5939,15 @@ static int ext4_try_to_trim_range(struct super_block *sb, struct ext4_buddy *e4b, ext4_grpblk_t start, ext4_grpblk_t max, ext4_grpblk_t minblocks) { - ext4_grpblk_t next, count, free_count; + ext4_grpblk_t next, count, free_count, last, origin_start; bool set_trimmed = false; void *bitmap; + last = ext4_last_grp_cluster(sb, e4b->bd_group); bitmap = e4b->bd_bitmap; - if (start == 0 && max >= ext4_last_grp_cluster(sb, e4b->bd_group)) + if (start == 0 && max >= last) set_trimmed = true; + origin_start = start; start = max(e4b->bd_info->bb_first_free, start); count = 0; free_count = 0; @@ -5949,7 +5956,10 @@ static int ext4_try_to_trim_range(struct super_block *sb, start = mb_find_next_zero_bit(bitmap, max + 1, start); if (start > max) break; - next = mb_find_next_bit(bitmap, max + 1, start); + + next = mb_find_next_bit(bitmap, last + 1, start); + if (origin_start == 0 && next >= last) + set_trimmed = true; if ((next - start) >= minblocks) { int ret = ext4_trim_extent(sb, start, next - start, e4b); diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c index 7b390320970f1c02ba5f57e7a8eb0267facdfd9a..f142537a81962247ad337c4577ca0e97e90844b2 100644 --- a/fs/ext4/resize.c +++ b/fs/ext4/resize.c @@ -244,7 +244,7 @@ struct ext4_new_flex_group_data { * * Returns NULL on failure otherwise address of the allocated structure. */ -static struct ext4_new_flex_group_data *alloc_flex_gd(unsigned long flexbg_size) +static struct ext4_new_flex_group_data *alloc_flex_gd(unsigned int flexbg_size) { struct ext4_new_flex_group_data *flex_gd; @@ -303,7 +303,7 @@ static void free_flex_gd(struct ext4_new_flex_group_data *flex_gd) */ static int ext4_alloc_group_tables(struct super_block *sb, struct ext4_new_flex_group_data *flex_gd, - int flexbg_size) + unsigned int flexbg_size) { struct ext4_new_group_data *group_data = flex_gd->groups; ext4_fsblk_t start_blk; @@ -404,12 +404,12 @@ static int ext4_alloc_group_tables(struct super_block *sb, group = group_data[0].group; printk(KERN_DEBUG "EXT4-fs: adding a flex group with " - "%d groups, flexbg size is %d:\n", flex_gd->count, + "%u groups, flexbg size is %u:\n", flex_gd->count, flexbg_size); for (i = 0; i < flex_gd->count; i++) { ext4_debug( - "adding %s group %u: %u blocks (%d free, %d mdata blocks)\n", + "adding %s group %u: %u blocks (%u free, %u mdata blocks)\n", ext4_bg_has_super(sb, group + i) ? "normal" : "no-super", group + i, group_data[i].blocks_count, @@ -1964,8 +1964,9 @@ int ext4_resize_fs(struct super_block *sb, ext4_fsblk_t n_blocks_count) ext4_fsblk_t o_blocks_count; ext4_fsblk_t n_blocks_count_retry = 0; unsigned long last_update_time = 0; - int err = 0, flexbg_size = 1 << sbi->s_log_groups_per_flex; + int err = 0; int meta_bg; + unsigned int flexbg_size = ext4_flex_bg_size(sbi); /* See if the device is actually as big as what was requested */ bh = ext4_sb_bread(sb, n_blocks_count - 1, 0); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 702ac7d98333707374dfc0f66d04a41b618183f7..29c24715a6325f5b4ed6ee23f09a7d412ca45a79 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -64,7 +64,7 @@ enum { struct f2fs_fault_info { atomic_t inject_ops; - unsigned int inject_rate; + int inject_rate; unsigned int inject_type; }; @@ -4167,10 +4167,14 @@ static inline bool f2fs_force_buffered_io(struct inode *inode, } #ifdef CONFIG_F2FS_FAULT_INJECTION -extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, - unsigned int type); +extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type); #else -#define f2fs_build_fault_attr(sbi, rate, type) do { } while (0) +static int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type) +{ + return 0; +} #endif static inline bool is_journalled_quota(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 80cb10d2c3b015814f20e4287d4584daed5a6658..7e9379106f34270cd76de4b937941c329ca66e28 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -42,7 +42,7 @@ static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); up_read(&F2FS_I(inode)->i_mmap_sem); - if (!ret) + if (ret & VM_FAULT_LOCKED) f2fs_update_iostat(F2FS_I_SB(inode), APP_MAPPED_READ_IO, F2FS_BLKSIZE); @@ -2816,6 +2816,11 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out; } + if (f2fs_compressed_file(src) || f2fs_compressed_file(dst)) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + ret = -EINVAL; if (pos_in + len > src->i_size || pos_in + len < pos_in) goto out_unlock; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 45306a108b0f3b6fe298f987ecbc804369229e24..530c54c1d9098ade32480ac2e5ddf09de2bc168b 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -71,21 +71,31 @@ const char *f2fs_fault_name[FAULT_MAX] = { [FAULT_WRITE_IO] = "write IO error", }; -void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, - unsigned int type) +int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type) { struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; if (rate) { + if (rate > INT_MAX) + return -EINVAL; atomic_set(&ffi->inject_ops, 0); - ffi->inject_rate = rate; + ffi->inject_rate = (int)rate; } - if (type) - ffi->inject_type = type; + if (type) { + if (type >= BIT(FAULT_MAX)) + return -EINVAL; + ffi->inject_type = (unsigned int)type; + } if (!rate && !type) memset(ffi, 0, sizeof(struct f2fs_fault_info)); + else + f2fs_info(sbi, + "build fault injection attr: rate: %lu, type: 0x%lx", + rate, type); + return 0; } #endif @@ -745,14 +755,17 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) case Opt_fault_injection: if (args->from && match_int(args, &arg)) return -EINVAL; - f2fs_build_fault_attr(sbi, arg, F2FS_ALL_FAULT_TYPE); + if (f2fs_build_fault_attr(sbi, arg, + F2FS_ALL_FAULT_TYPE)) + return -EINVAL; set_opt(sbi, FAULT_INJECTION); break; case Opt_fault_type: if (args->from && match_int(args, &arg)) return -EINVAL; - f2fs_build_fault_attr(sbi, 0, arg); + if (f2fs_build_fault_attr(sbi, 0, arg)) + return -EINVAL; set_opt(sbi, FAULT_INJECTION); break; #else @@ -1739,8 +1752,6 @@ static void default_options(struct f2fs_sb_info *sbi) F2FS_OPTION(sbi).compress_ext_cnt = 0; F2FS_OPTION(sbi).bggc_mode = BGGC_MODE_ON; - sbi->sb->s_flags &= ~SB_INLINECRYPT; - set_opt(sbi, INLINE_XATTR); set_opt(sbi, INLINE_DATA); set_opt(sbi, INLINE_DENTRY); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 6f68b20c1f9af18b2b57e4a5499e39973249bed8..29b069e9abd5643e53ee5701c4f19609e2214e05 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -329,10 +329,16 @@ static ssize_t __sbi_store(struct f2fs_attr *a, if (ret < 0) return ret; #ifdef CONFIG_F2FS_FAULT_INJECTION - if (a->struct_type == FAULT_INFO_TYPE && t >= (1 << FAULT_MAX)) - return -EINVAL; - if (a->struct_type == FAULT_INFO_RATE && t >= UINT_MAX) - return -EINVAL; + if (a->struct_type == FAULT_INFO_TYPE) { + if (f2fs_build_fault_attr(sbi, 0, t)) + return -EINVAL; + return count; + } + if (a->struct_type == FAULT_INFO_RATE) { + if (f2fs_build_fault_attr(sbi, t, 0)) + return -EINVAL; + return count; + } #endif if (a->struct_type == RESERVED_BLOCKS) { spin_lock(&sbi->stat_lock); diff --git a/fs/file.c b/fs/file.c index d6bc73960e4ac9df2258a4344ea5ab386965887c..e66636f962c0f4b942e24a0ee9841a9cf244dda1 100644 --- a/fs/file.c +++ b/fs/file.c @@ -985,7 +985,7 @@ static unsigned long __fget_light(unsigned int fd, fmode_t mask) struct file *file; if (atomic_read(&files->count) == 1) { - file = __fcheck_files(files, fd); + file = files_lookup_fd_raw(files, fd); if (!file || unlikely(file->f_mode & mask)) return 0; return (unsigned long)file; @@ -1210,7 +1210,7 @@ static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags) spin_lock(&files->file_lock); err = expand_files(files, newfd); - file = fcheck(oldfd); + file = files_lookup_fd_locked(files, oldfd); if (unlikely(!file)) goto Ebadf; if (unlikely(err < 0)) { diff --git a/fs/ioctl.c b/fs/ioctl.c index 7bcc60091287c75460c8713267c355ce6bdb0a7a..b2d06f016ec6624beb25d82c208d7559da8f8fdc 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -799,8 +799,7 @@ COMPAT_SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, if (!f.file) return -EBADF; - /* RED-PEN how should LSM module know it's handling 32bit? */ - error = security_file_ioctl(f.file, cmd, arg); + error = security_file_ioctl_compat(f.file, cmd, arg); if (error) goto out; diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c index 81ca58c10b728cc6a7c4e132a80c503c0dde1df3..40cc5e62907c1e50993cc350a2f8213377807024 100644 --- a/fs/jffs2/super.c +++ b/fs/jffs2/super.c @@ -58,6 +58,7 @@ static void jffs2_i_init_once(void *foo) struct jffs2_inode_info *f = foo; mutex_init(&f->sem); + f->target = NULL; inode_init_once(&f->vfs_inode); } diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c index 72eb5ed54c2abac0e419d061ebf368a498ada85d..9b6849b9bfdb9d5b44865aeedd9d3912a8cd4111 100644 --- a/fs/jfs/jfs_dmap.c +++ b/fs/jfs/jfs_dmap.c @@ -63,10 +63,10 @@ */ static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno, int nblocks); -static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval); -static int dbBackSplit(dmtree_t * tp, int leafno); -static int dbJoin(dmtree_t * tp, int leafno, int newval); -static void dbAdjTree(dmtree_t * tp, int leafno, int newval); +static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl); +static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl); +static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl); +static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl); static int dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level); static int dbAllocAny(struct bmap * bmp, s64 nblocks, int l2nb, s64 * results); @@ -2171,7 +2171,7 @@ static int dbFreeDmap(struct bmap * bmp, struct dmap * dp, s64 blkno, * system. */ if (dp->tree.stree[word] == NOFREE) - dbBackSplit((dmtree_t *) & dp->tree, word); + dbBackSplit((dmtree_t *)&dp->tree, word, false); dbAllocBits(bmp, dp, blkno, nblocks); } @@ -2257,7 +2257,7 @@ static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno, * the binary system of the leaves if need be. */ dbSplit(tp, word, BUDMIN, - dbMaxBud((u8 *) & dp->wmap[word])); + dbMaxBud((u8 *)&dp->wmap[word]), false); word += 1; } else { @@ -2297,7 +2297,7 @@ static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno, * system of the leaves to reflect the current * allocation (size). */ - dbSplit(tp, word, size, NOFREE); + dbSplit(tp, word, size, NOFREE, false); /* get the number of dmap words handled */ nw = BUDSIZE(size, BUDMIN); @@ -2404,7 +2404,7 @@ static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno, /* update the leaf for this dmap word. */ rc = dbJoin(tp, word, - dbMaxBud((u8 *) & dp->wmap[word])); + dbMaxBud((u8 *)&dp->wmap[word]), false); if (rc) return rc; @@ -2437,7 +2437,7 @@ static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno, /* update the leaf. */ - rc = dbJoin(tp, word, size); + rc = dbJoin(tp, word, size, false); if (rc) return rc; @@ -2589,14 +2589,14 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level) * that it is at the front of a binary buddy system. */ if (oldval == NOFREE) { - rc = dbBackSplit((dmtree_t *) dcp, leafno); + rc = dbBackSplit((dmtree_t *)dcp, leafno, true); if (rc) return rc; oldval = dcp->stree[ti]; } - dbSplit((dmtree_t *) dcp, leafno, dcp->budmin, newval); + dbSplit((dmtree_t *) dcp, leafno, dcp->budmin, newval, true); } else { - rc = dbJoin((dmtree_t *) dcp, leafno, newval); + rc = dbJoin((dmtree_t *) dcp, leafno, newval, true); if (rc) return rc; } @@ -2625,7 +2625,7 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level) */ if (alloc) { dbJoin((dmtree_t *) dcp, leafno, - oldval); + oldval, true); } else { /* the dbJoin() above might have * caused a larger binary buddy system @@ -2635,9 +2635,9 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level) */ if (dcp->stree[ti] == NOFREE) dbBackSplit((dmtree_t *) - dcp, leafno); + dcp, leafno, true); dbSplit((dmtree_t *) dcp, leafno, - dcp->budmin, oldval); + dcp->budmin, oldval, true); } /* release the buffer and return the error. @@ -2685,7 +2685,7 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level) * * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; */ -static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval) +static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl) { int budsz; int cursz; @@ -2707,7 +2707,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval) while (cursz >= splitsz) { /* update the buddy's leaf with its new value. */ - dbAdjTree(tp, leafno ^ budsz, cursz); + dbAdjTree(tp, leafno ^ budsz, cursz, is_ctl); /* on to the next size and buddy. */ @@ -2719,7 +2719,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval) /* adjust the dmap tree to reflect the specified leaf's new * value. */ - dbAdjTree(tp, leafno, newval); + dbAdjTree(tp, leafno, newval, is_ctl); } @@ -2750,7 +2750,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval) * * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; */ -static int dbBackSplit(dmtree_t * tp, int leafno) +static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl) { int budsz, bud, w, bsz, size; int cursz; @@ -2801,7 +2801,7 @@ static int dbBackSplit(dmtree_t * tp, int leafno) * system in two. */ cursz = leaf[bud] - 1; - dbSplit(tp, bud, cursz, cursz); + dbSplit(tp, bud, cursz, cursz, is_ctl); break; } } @@ -2829,7 +2829,7 @@ static int dbBackSplit(dmtree_t * tp, int leafno) * * RETURN VALUES: none */ -static int dbJoin(dmtree_t * tp, int leafno, int newval) +static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl) { int budsz, buddy; s8 *leaf; @@ -2884,12 +2884,12 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval) if (leafno < buddy) { /* leafno is the left buddy. */ - dbAdjTree(tp, buddy, NOFREE); + dbAdjTree(tp, buddy, NOFREE, is_ctl); } else { /* buddy is the left buddy and becomes * leafno. */ - dbAdjTree(tp, leafno, NOFREE); + dbAdjTree(tp, leafno, NOFREE, is_ctl); leafno = buddy; } @@ -2902,7 +2902,7 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval) /* update the leaf value. */ - dbAdjTree(tp, leafno, newval); + dbAdjTree(tp, leafno, newval, is_ctl); return 0; } @@ -2923,15 +2923,20 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval) * * RETURN VALUES: none */ -static void dbAdjTree(dmtree_t * tp, int leafno, int newval) +static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl) { int lp, pp, k; - int max; + int max, size; + + size = is_ctl ? CTLTREESIZE : TREESIZE; /* pick up the index of the leaf for this leafno. */ lp = leafno + le32_to_cpu(tp->dmt_leafidx); + if (WARN_ON_ONCE(lp >= size || lp < 0)) + return; + /* is the current value the same as the old value ? if so, * there is nothing to do. */ diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 837d42f61464b3a7e06bd8cf625d8e1a96fbfd5e..a222a9d71887f564c61fd599c974207941a39493 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -633,6 +633,11 @@ int dtSearch(struct inode *ip, struct component_name * key, ino_t * data, for (base = 0, lim = p->header.nextindex; lim; lim >>= 1) { index = base + (lim >> 1); + if (stbl[index] < 0) { + rc = -EIO; + goto out; + } + if (p->header.flag & BT_LEAF) { /* uppercase leaf name to compare */ cmp = @@ -1970,7 +1975,7 @@ static int dtSplitRoot(tid_t tid, do { f = &rp->slot[fsi]; fsi = f->next; - } while (fsi != -1); + } while (fsi >= 0); f->next = n; } diff --git a/fs/jfs/jfs_imap.c b/fs/jfs/jfs_imap.c index 14f918a4831d354df4dbd3776f06a90cd2fd0725..b0965f3ef18658ac438fe3ae6af80e1f50d2c75a 100644 --- a/fs/jfs/jfs_imap.c +++ b/fs/jfs/jfs_imap.c @@ -2181,6 +2181,9 @@ static int diNewExt(struct inomap * imap, struct iag * iagp, int extno) /* get the ag and iag numbers for this iag. */ agno = BLKTOAG(le64_to_cpu(iagp->agstart), sbi); + if (agno >= MAXAG || agno < 0) + return -EIO; + iagno = le32_to_cpu(iagp->iagnum); /* check if this is the last free extent within the diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c index aa4ff7bcaff23dd8931de14fbd629467819772e0..55702b31ab3c41c8bf28460349b0b60fdbf5ec32 100644 --- a/fs/jfs/jfs_mount.c +++ b/fs/jfs/jfs_mount.c @@ -172,15 +172,15 @@ int jfs_mount(struct super_block *sb) } jfs_info("jfs_mount: ipimap:0x%p", ipimap); - /* map further access of per fileset inodes by the fileset inode */ - sbi->ipimap = ipimap; - /* initialize fileset inode allocation map */ if ((rc = diMount(ipimap))) { jfs_err("jfs_mount: diMount failed w/rc = %d", rc); goto err_ipimap; } + /* map further access of per fileset inodes by the fileset inode */ + sbi->ipimap = ipimap; + return rc; /* diff --git a/fs/locks.c b/fs/locks.c index 4e4b36c330f96e7859379b9d44750120729c5278..af603e103e9b2a0a93838357d057c12e3891e5e9 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -2530,24 +2530,24 @@ int fcntl_setlk(unsigned int fd, struct file *filp, unsigned int cmd, error = do_lock_file_wait(filp, cmd, file_lock); /* - * Attempt to detect a close/fcntl race and recover by releasing the - * lock that was just acquired. There is no need to do that when we're + * Detect close/fcntl races and recover by zapping all POSIX locks + * associated with this file and our files_struct, just like on + * filp_flush(). There is no need to do that when we're * unlocking though, or for OFD locks. */ if (!error && file_lock->fl_type != F_UNLCK && !(file_lock->fl_flags & FL_OFDLCK)) { + struct files_struct *files = current->files; /* * We need that spin_lock here - it prevents reordering between * update of i_flctx->flc_posix and check for it done in * close(). rcu_read_lock() wouldn't do. */ - spin_lock(¤t->files->file_lock); - f = fcheck(fd); - spin_unlock(¤t->files->file_lock); + spin_lock(&files->file_lock); + f = files_lookup_fd_locked(files, fd); + spin_unlock(&files->file_lock); if (f != filp) { - file_lock->fl_type = F_UNLCK; - error = do_lock_file_wait(filp, cmd, file_lock); - WARN_ON_ONCE(error); + locks_remove_posix(filp, files); error = -EBADF; } } @@ -2661,24 +2661,24 @@ int fcntl_setlk64(unsigned int fd, struct file *filp, unsigned int cmd, error = do_lock_file_wait(filp, cmd, file_lock); /* - * Attempt to detect a close/fcntl race and recover by releasing the - * lock that was just acquired. There is no need to do that when we're + * Detect close/fcntl races and recover by zapping all POSIX locks + * associated with this file and our files_struct, just like on + * filp_flush(). There is no need to do that when we're * unlocking though, or for OFD locks. */ if (!error && file_lock->fl_type != F_UNLCK && !(file_lock->fl_flags & FL_OFDLCK)) { + struct files_struct *files = current->files; /* * We need that spin_lock here - it prevents reordering between * update of i_flctx->flc_posix and check for it done in * close(). rcu_read_lock() wouldn't do. */ - spin_lock(¤t->files->file_lock); - f = fcheck(fd); - spin_unlock(¤t->files->file_lock); + spin_lock(&files->file_lock); + f = files_lookup_fd_locked(files, fd); + spin_unlock(&files->file_lock); if (f != filp) { - file_lock->fl_type = F_UNLCK; - error = do_lock_file_wait(filp, cmd, file_lock); - WARN_ON_ONCE(error); + locks_remove_posix(filp, files); error = -EBADF; } } diff --git a/fs/namei.c b/fs/namei.c index 0c64ec99b8b998029370d6a0bc2835b44e6810e2..a66e1910f4026ec6782f8267368ccea301c21676 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2778,20 +2778,14 @@ struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) p = d_ancestor(p2, p1); if (p) { inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); - inode_lock_nested(p1->d_inode, I_MUTEX_CHILD); + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT2); return p; } p = d_ancestor(p1, p2); - if (p) { - inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); - inode_lock_nested(p2->d_inode, I_MUTEX_CHILD); - return p; - } - - lock_two_inodes(p1->d_inode, p2->d_inode, - I_MUTEX_PARENT, I_MUTEX_PARENT2); - return NULL; + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2); + return p; } EXPORT_SYMBOL(lock_rename); @@ -4266,11 +4260,12 @@ SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname * * a) we can get into loop creation. * b) race potential - two innocent renames can create a loop together. - * That's where 4.4 screws up. Current fix: serialization on + * That's where 4.4BSD screws up. Current fix: serialization on * sb->s_vfs_rename_mutex. We might be more accurate, but that's another * story. - * c) we have to lock _four_ objects - parents and victim (if it exists), - * and source. + * c) we may have to lock up to _four_ objects - parents and victim (if it exists), + * and source (if it's a non-directory or a subdirectory that moves to + * different parent). * And that - after we got ->i_mutex on parents (until then we don't know * whether the target exists). Solution: try to be smart with locking * order for inodes. We rely on the fact that tree topology may change @@ -4299,6 +4294,7 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, bool new_is_dir = false; unsigned max_links = new_dir->i_sb->s_max_links; struct name_snapshot old_name; + bool lock_old_subdir, lock_new_subdir; if (source == target) return 0; @@ -4348,15 +4344,32 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, take_dentry_name_snapshot(&old_name, old_dentry); dget(new_dentry); /* - * Lock all moved children. Moved directories may need to change parent - * pointer so they need the lock to prevent against concurrent - * directory changes moving parent pointer. For regular files we've - * historically always done this. The lockdep locking subclasses are - * somewhat arbitrary but RENAME_EXCHANGE in particular can swap - * regular files and directories so it's difficult to tell which - * subclasses to use. + * Lock children. + * The source subdirectory needs to be locked on cross-directory + * rename or cross-directory exchange since its parent changes. + * The target subdirectory needs to be locked on cross-directory + * exchange due to parent change and on any rename due to becoming + * a victim. + * Non-directories need locking in all cases (for NFS reasons); + * they get locked after any subdirectories (in inode address order). + * + * NOTE: WE ONLY LOCK UNRELATED DIRECTORIES IN CROSS-DIRECTORY CASE. + * NEVER, EVER DO THAT WITHOUT ->s_vfs_rename_mutex. */ - lock_two_inodes(source, target, I_MUTEX_NORMAL, I_MUTEX_NONDIR2); + lock_old_subdir = new_dir != old_dir; + lock_new_subdir = new_dir != old_dir || !(flags & RENAME_EXCHANGE); + if (is_dir) { + if (lock_old_subdir) + inode_lock_nested(source, I_MUTEX_CHILD); + if (target && (!new_is_dir || lock_new_subdir)) + inode_lock(target); + } else if (new_is_dir) { + if (lock_new_subdir) + inode_lock_nested(target, I_MUTEX_CHILD); + inode_lock(source); + } else { + lock_two_nondirectories(source, target); + } error = -EBUSY; if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry)) @@ -4400,8 +4413,9 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, d_exchange(old_dentry, new_dentry); } out: - inode_unlock(source); - if (target) + if (!is_dir || lock_old_subdir) + inode_unlock(source); + if (target && (!new_is_dir || lock_new_subdir)) inode_unlock(target); dput(new_dentry); if (!error) { diff --git a/fs/open.c b/fs/open.c index dde3415cbc8a9cba6a1ebb1dd89c18dcec86e02f..2acd46af38aefc215129749d8cc12713fde6e032 100644 --- a/fs/open.c +++ b/fs/open.c @@ -200,13 +200,13 @@ long do_sys_ftruncate(unsigned int fd, loff_t length, int small) return error; } -SYSCALL_DEFINE2(ftruncate, unsigned int, fd, unsigned long, length) +SYSCALL_DEFINE2(ftruncate, unsigned int, fd, off_t, length) { return do_sys_ftruncate(fd, length, 1); } #ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE2(ftruncate, unsigned int, fd, compat_ulong_t, length) +COMPAT_SYSCALL_DEFINE2(ftruncate, unsigned int, fd, compat_off_t, length) { return do_sys_ftruncate(fd, length, 1); } diff --git a/fs/proc/fd.c b/fs/proc/fd.c index 81882a13212d3dd7d9c5cffb8e12340f7e8362c6..6640ec2e5c48ee4be58ab45cc8c437aa59b0fc6d 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -35,7 +35,7 @@ static int seq_show(struct seq_file *m, void *v) unsigned int fd = proc_fd(m->private); spin_lock(&files->file_lock); - file = fcheck_files(files, fd); + file = files_lookup_fd_locked(files, fd); if (file) { struct fdtable *fdt = files_fdtable(files); diff --git a/fs/sharefs/authentication.c b/fs/sharefs/authentication.c index 8986fcbcc894dd48f5312a6b77d2d6327b077089..bc6e825092fa5e029cd6990562bc635e8ba8f1d1 100644 --- a/fs/sharefs/authentication.c +++ b/fs/sharefs/authentication.c @@ -75,7 +75,8 @@ void sharefs_root_inode_perm_init(struct inode *root_inode) hii->perm = SHAREFS_PERM_FIX; } -const struct cred *sharefs_override_file_fsids(struct inode *dir) +#ifdef CONFIG_SHAREFS_SUPPORT_OVERRIDE +const struct cred *sharefs_override_file_fsids(struct inode *dir, __u16 *_perm) { struct cred *cred = NULL; cred = prepare_creds(); @@ -94,3 +95,4 @@ void sharefs_revert_fsids(const struct cred *old_cred) revert_creds(old_cred); put_cred(cur_cred); } +#endif diff --git a/fs/sharefs/authentication.h b/fs/sharefs/authentication.h index 0a096d1ee710413580719544ec81cf731bdb86de..e598fa883c0a47f52b4264314864ccad4d2884fb 100644 --- a/fs/sharefs/authentication.h +++ b/fs/sharefs/authentication.h @@ -40,8 +40,11 @@ extern void sharefs_exit_configfs(void); void sharefs_root_inode_perm_init(struct inode *root_inode); void fixup_perm_from_level(struct inode *dir, struct dentry *dentry); -const struct cred *sharefs_override_file_fsids(struct inode *dir); +#ifdef CONFIG_SHAREFS_SUPPORT_OVERRIDE +const struct cred *sharefs_override_file_fsids(struct inode *dir, + __u16 *_perm); void sharefs_revert_fsids(const struct cred *old_cred); +#endif static inline bool is_read_only_auth(__u16 perm) { diff --git a/fs/sharefs/dentry.c b/fs/sharefs/dentry.c index c1a4c30f30fe70596621bca66c882090b2c4c914..dee29cace6b751a5cd29b648bb5024bc53e96714 100644 --- a/fs/sharefs/dentry.c +++ b/fs/sharefs/dentry.c @@ -10,7 +10,6 @@ */ #include "sharefs.h" -#include "authentication.h" /* * returns: 0: tell VFS to invalidate dentry in share directory @@ -30,7 +29,7 @@ static void sharefs_d_release(struct dentry *dentry) */ if (SHAREFS_D(dentry)) { /* release and reset the lower paths */ - sharefs_reset_lower_path(dentry); + sharefs_put_reset_lower_path(dentry); free_dentry_private_data(dentry); } return; @@ -40,106 +39,3 @@ const struct dentry_operations sharefs_dops = { .d_revalidate = sharefs_d_revalidate, .d_release = sharefs_d_release, }; - -static int try_to_create_lower_path(struct dentry *d, struct path *lower_path) -{ - int err; - struct path lower_dir_path; - struct qstr this; - struct dentry *parent; - struct dentry *lower_dentry; - struct inode *lower_dir_inode; - - parent = dget_parent(d); - err = sharefs_get_lower_path(parent, &lower_dir_path, 0); - dput(parent); - if (err) - return err; - /* instantiate a new negative dentry */ - this.name = d->d_name.name; - this.len = strlen(d->d_name.name); - this.hash = full_name_hash(lower_dir_path.dentry, this.name, this.len); - lower_dentry = d_alloc(lower_dir_path.dentry, &this); - if (!lower_dentry) { - err = -ENOMEM; - goto out; - } - lower_dir_inode = d_inode(lower_dir_path.dentry); - if (lower_dir_inode->i_op && lower_dir_inode->i_op->lookup) - lower_dir_inode->i_op->lookup(lower_dir_inode, lower_dentry, 0); - - spin_lock(&SHAREFS_D(d)->lock); - lower_path->dentry = lower_dentry; - lower_path->mnt = lower_dir_path.mnt; - spin_unlock(&SHAREFS_D(d)->lock); - -out: - sharefs_put_lower_path(parent, &lower_dir_path); - return err; -} - -/* Returns struct path. Caller must path_put it. */ -int sharefs_get_lower_path(struct dentry *d, struct path *lower_path, - bool try_to_create) -{ - int err = -ENOMEM; - char *path_buf; - char *path_name = NULL; - struct path lower_root_path; - const struct cred *saved_cred = NULL; - - if (unlikely(!d)) - goto out; - - path_buf = kmalloc(PATH_MAX, GFP_KERNEL); - if (unlikely(!path_buf)) - goto out; - path_name = dentry_path_raw(d, path_buf, PATH_MAX); - if (IS_ERR(path_name)) { - err = PTR_ERR(path_name); - goto out_free; - } - - sharefs_get_lower_root_path(d, &lower_root_path); - saved_cred = sharefs_override_file_fsids(d_inode(lower_root_path.dentry)); - if (!saved_cred) { - sharefs_put_lower_path(d, &lower_root_path); - goto out_free; - } - spin_lock(&SHAREFS_D(d)->lock); - err = vfs_path_lookup(lower_root_path.dentry, lower_root_path.mnt, - path_name, LOOKUP_CREATE, lower_path); - spin_unlock(&SHAREFS_D(d)->lock); - sharefs_revert_fsids(saved_cred); - sharefs_put_lower_path(d, &lower_root_path); - - if (err != -ENOENT || !try_to_create) - goto out_free; - err = try_to_create_lower_path(d, lower_path); - -out_free: - kfree(path_buf); -out: - return err; -} - -int sharefs_get_lower_inode(struct dentry *d, struct inode **lower_inode) -{ - int err = 0; - struct path lower_path; - - err = sharefs_get_lower_path(d, &lower_path, 0); - if (err) - goto out; - - *lower_inode = d_inode(lower_path.dentry); - if ((*lower_inode)->i_flags & S_DEAD) { - err = -ENOENT; - } else if (!igrab(*lower_inode)) { - err = -ESTALE; - } - - sharefs_put_lower_path(d, &lower_path); -out: - return err; -} \ No newline at end of file diff --git a/fs/sharefs/file.c b/fs/sharefs/file.c index 1aa44baf67e5d0e1a848e24c04c739130c173a5b..f04963d57a4a6b8896393ade9d80d59cae885ff9 100644 --- a/fs/sharefs/file.c +++ b/fs/sharefs/file.c @@ -31,7 +31,6 @@ static int sharefs_open(struct inode *inode, struct file *file) int err = 0; struct file *lower_file = NULL; struct path lower_path; - struct inode *lower_inode = NULL; /* don't open unhashed/deleted files */ if (d_unhashed(file->f_path.dentry)) { @@ -47,9 +46,7 @@ static int sharefs_open(struct inode *inode, struct file *file) } /* open lower object and link sharefs's file struct to lower's */ - err = sharefs_get_lower_path(file->f_path.dentry, &lower_path, 0); - if (err) - goto out_free; + sharefs_get_lower_path(file->f_path.dentry, &lower_path); lower_file = dentry_open(&lower_path, file->f_flags, current_cred()); path_put(&lower_path); if (IS_ERR(lower_file)) { @@ -63,21 +60,16 @@ static int sharefs_open(struct inode *inode, struct file *file) sharefs_set_lower_file(file, lower_file); } -out_free: if (err) { kfree(SHAREFS_F(file)); } else { kuid_t uid = inode->i_uid; kgid_t gid = inode->i_gid; mode_t mode = inode->i_mode; - err = sharefs_get_lower_inode(file->f_path.dentry, &lower_inode); - if (err) - goto out_err; - fsstack_copy_attr_all(inode, lower_inode); + fsstack_copy_attr_all(inode, sharefs_lower_inode(inode)); inode->i_uid = uid; inode->i_gid = gid; inode->i_mode = mode; - iput(lower_inode); } out_err: return err; @@ -124,9 +116,7 @@ static int sharefs_fsync(struct file *file, loff_t start, loff_t end, if (err) goto out; lower_file = sharefs_lower_file(file); - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - goto out; + sharefs_get_lower_path(dentry, &lower_path); err = vfs_fsync_range(lower_file, start, end, datasync); sharefs_put_lower_path(dentry, &lower_path); out: diff --git a/fs/sharefs/inode.c b/fs/sharefs/inode.c index e5cb396263e219dab9262df74c1dc380a2e720f5..7bd08abe79ea30ad6ec60298f812f00837176702 100644 --- a/fs/sharefs/inode.c +++ b/fs/sharefs/inode.c @@ -20,9 +20,7 @@ static int sharefs_getattr(const struct path *path, struct kstat *stat, struct path lower_path; int ret; - ret = sharefs_get_lower_path(path->dentry, &lower_path, 0); - if (ret) - return ret; + sharefs_get_lower_path(path->dentry, &lower_path); ret = vfs_getattr(&lower_path, stat, request_mask, flags); stat->ino = d_inode(path->dentry)->i_ino; stat->uid = d_inode(path->dentry)->i_uid; @@ -41,14 +39,8 @@ static ssize_t sharefs_listxattr(struct dentry *dentry, char *buffer, size_t buf struct dentry *lower_dentry; struct path lower_path; - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - return err; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; - if (d_inode(lower_dentry)->i_flags & S_DEAD) { - err = -ENOENT; - goto out; - } if (!(d_inode(lower_dentry)->i_opflags & IOP_XATTR)) { err = -EOPNOTSUPP; goto out; @@ -93,23 +85,17 @@ static int sharefs_create(struct inode *dir, struct dentry *dentry, struct dentry *lower_parent_dentry = NULL; struct path lower_path; const struct cred *saved_cred = NULL; - struct inode *lower_inode = NULL; + __u16 child_perm; - saved_cred = sharefs_override_file_fsids(dir); + saved_cred = sharefs_override_file_fsids(dir, &child_perm); if (!saved_cred) { err = -ENOMEM; return err; } - err = sharefs_get_lower_path(dentry, &lower_path, 1); - if (err) - goto out_revert; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; lower_parent_dentry = lock_parent(lower_dentry); - if (unlikely(!lower_parent_dentry)) { - err = -ENOENT; - goto out; - } err = vfs_create(d_inode(lower_parent_dentry), lower_dentry, mode, want_excl); if (err) @@ -117,17 +103,12 @@ static int sharefs_create(struct inode *dir, struct dentry *dentry, err = sharefs_interpose(dentry, dir->i_sb, &lower_path); if (err) goto out; - err = sharefs_get_lower_inode(dentry, &lower_inode); - if (err) - goto out; - fsstack_copy_attr_times(dir, lower_inode); + fsstack_copy_attr_times(dir, sharefs_lower_inode(dir)); fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); - iput(lower_inode); out: unlock_dir(lower_parent_dentry); sharefs_put_lower_path(dentry, &lower_path); -out_revert: sharefs_revert_fsids(saved_cred); return err; } @@ -139,23 +120,17 @@ static int sharefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) struct dentry *lower_parent_dentry = NULL; struct path lower_path; const struct cred *saved_cred = NULL; - struct inode *lower_inode = NULL; + __u16 child_perm; - saved_cred = sharefs_override_file_fsids(dir); + saved_cred = sharefs_override_file_fsids(dir, &child_perm); if (!saved_cred) { err = -ENOMEM; return err; } - err = sharefs_get_lower_path(dentry, &lower_path, 1); - if (err) - goto out_revert; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; lower_parent_dentry = lock_parent(lower_dentry); - if (unlikely(!lower_parent_dentry)) { - err = -ENOENT; - goto out; - } err = vfs_mkdir(d_inode(lower_parent_dentry), lower_dentry, mode); if (err) goto out; @@ -164,19 +139,14 @@ static int sharefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) if (err) goto out; - err = sharefs_get_lower_inode(dentry, &lower_inode); - if (err) - goto out; - fsstack_copy_attr_times(dir, lower_inode); + fsstack_copy_attr_times(dir, sharefs_lower_inode(dir)); fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); /* update number of links on parent directory */ - set_nlink(dir, lower_inode->i_nlink); - iput(lower_inode); + set_nlink(dir, sharefs_lower_inode(dir)->i_nlink); out: unlock_dir(lower_parent_dentry); sharefs_put_lower_path(dentry, &lower_path); -out_revert: sharefs_revert_fsids(saved_cred); return err; } @@ -185,20 +155,11 @@ static int sharefs_unlink(struct inode *dir, struct dentry *dentry) { int err; struct dentry *lower_dentry = NULL; - struct inode *lower_dir_inode = NULL; - struct inode *lower_inode = NULL; + struct inode *lower_dir_inode = sharefs_lower_inode(dir); struct dentry *lower_dir_dentry = NULL; struct path lower_path; - err = sharefs_get_lower_inode(dentry->d_parent, &lower_dir_inode); - if (err) - return err; - err = sharefs_get_lower_inode(dentry, &lower_inode); - if (err) - goto put; - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - goto put_dir; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; dget(lower_dentry); lower_dir_dentry = lock_parent(lower_dentry); @@ -207,7 +168,8 @@ static int sharefs_unlink(struct inode *dir, struct dentry *dentry) goto out; fsstack_copy_attr_times(dir, lower_dir_inode); fsstack_copy_inode_size(dir, lower_dir_inode); - set_nlink(dentry->d_inode, lower_inode->i_nlink); + set_nlink(dentry->d_inode, + sharefs_lower_inode(dentry->d_inode)->i_nlink); dentry->d_inode->i_ctime = dir->i_ctime; d_drop(dentry); @@ -215,10 +177,6 @@ static int sharefs_unlink(struct inode *dir, struct dentry *dentry) unlock_dir(lower_dir_dentry); dput(lower_dentry); sharefs_put_lower_path(dentry, &lower_path); -put_dir: - iput(lower_inode); -put: - iput(lower_dir_inode); return err; } @@ -229,9 +187,7 @@ static int sharefs_rmdir(struct inode *dir, struct dentry *dentry) struct dentry *lower_dir_dentry; struct path lower_path; - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - return err; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; lower_dir_dentry = lock_parent(lower_dentry); err = vfs_rmdir(lower_dir_dentry->d_inode, lower_dentry); @@ -266,14 +222,8 @@ static int sharefs_rename(struct inode *old_dir, struct dentry *old_dentry, if (flags) return -EINVAL; - err = sharefs_get_lower_path(old_dentry, &lower_old_path, 0); - if (err) - return err; - err = sharefs_get_lower_path(new_dentry, &lower_new_path, 1); - if (err) { - sharefs_put_lower_path(old_dentry, &lower_old_path); - return err; - } + sharefs_get_lower_path(old_dentry, &lower_old_path); + sharefs_get_lower_path(new_dentry, &lower_new_path); lower_old_dentry = lower_old_path.dentry; lower_new_dentry = lower_new_path.dentry; lower_old_dir_dentry = dget_parent(lower_old_dentry); @@ -332,15 +282,11 @@ static int sharefs_setattr(struct dentry *dentry, struct iattr *ia) err = setattr_prepare(dentry, ia); if (err) - return err; + goto out_err; - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - return err; + sharefs_get_lower_path(dentry, &lower_path); lower_dentry = lower_path.dentry; - err = sharefs_get_lower_inode(dentry, &lower_inode); - if (err) - goto out_err; + lower_inode = sharefs_lower_inode(inode); /* prepare our own lower struct iattr (with the lower file) */ memcpy(&lower_ia, ia, sizeof(lower_ia)); @@ -393,9 +339,8 @@ static int sharefs_setattr(struct dentry *dentry, struct iattr *ia) */ out: - iput(lower_inode); -out_err: sharefs_put_lower_path(dentry, &lower_path); +out_err: return err; } #endif diff --git a/fs/sharefs/lookup.c b/fs/sharefs/lookup.c index 9d5536c83161854d9e8a79ed337b5a5e6a74d977..0b0c30ef46f12aa9e22c49ebe7cee8ca5efd9875 100644 --- a/fs/sharefs/lookup.c +++ b/fs/sharefs/lookup.c @@ -141,7 +141,6 @@ struct inode *sharefs_iget(struct super_block *sb, struct inode *lower_inode) fsstack_copy_inode_size(inode, lower_inode); unlock_new_inode(inode); - iput(lower_inode); return inode; } @@ -200,14 +199,14 @@ int sharefs_interpose(struct dentry *dentry, struct super_block *sb, */ static struct dentry *__sharefs_lookup(struct dentry *dentry, unsigned int flags, - struct path *lower_parent_path, - struct path *lower_path) + struct path *lower_parent_path) { int err = 0; struct vfsmount *lower_dir_mnt; struct dentry *lower_dir_dentry = NULL; struct dentry *lower_dentry; const char *name; + struct path lower_path; struct qstr this; struct dentry *ret_dentry = NULL; @@ -215,7 +214,7 @@ static struct dentry *__sharefs_lookup(struct dentry *dentry, d_set_d_op(dentry, &sharefs_dops); if (IS_ROOT(dentry)) - return NULL; + goto out; name = dentry->d_name.name; @@ -225,12 +224,12 @@ static struct dentry *__sharefs_lookup(struct dentry *dentry, /* Use vfs_path_lookup to check if the dentry exists or not */ err = vfs_path_lookup(lower_dir_dentry, lower_dir_mnt, name, 0, - lower_path); + &lower_path); /* no error: handle positive dentries */ if (!err) { - sharefs_set_lower_path(dentry, lower_path); + sharefs_set_lower_path(dentry, &lower_path); ret_dentry = - __sharefs_interpose(dentry, dentry->d_sb, lower_path); + __sharefs_interpose(dentry, dentry->d_sb, &lower_path); if (IS_ERR(ret_dentry)) { err = PTR_ERR(ret_dentry); /* path_put underlying path on error */ @@ -268,9 +267,9 @@ static struct dentry *__sharefs_lookup(struct dentry *dentry, lower_dentry, flags); setup_lower: - lower_path->dentry = lower_dentry; - lower_path->mnt = mntget(lower_dir_mnt); - sharefs_set_lower_path(dentry, lower_path); + lower_path.dentry = lower_dentry; + lower_path.mnt = mntget(lower_dir_mnt); + sharefs_set_lower_path(dentry, &lower_path); /* * If the intent is to create a file, then don't return an error, so @@ -291,31 +290,29 @@ struct dentry *sharefs_lookup(struct inode *dir, struct dentry *dentry, { int err; struct dentry *ret, *parent; - struct path lower_path; struct path lower_parent_path; #ifdef CONFIG_SHAREFS_SUPPORT_OVERRIDE const struct cred *saved_cred = NULL; + __u16 permission; #endif + parent = dget_parent(dentry); - err = sharefs_get_lower_path(parent, &lower_parent_path, 0); - if (err) { - dput(parent); - return ERR_PTR(err); - } + sharefs_get_lower_path(parent, &lower_parent_path); #ifdef CONFIG_SHAREFS_SUPPORT_OVERRIDE - saved_cred = sharefs_override_file_fsids(dir); + saved_cred = sharefs_override_file_fsids(dir, &permission); if (!saved_cred) { ret = ERR_PTR(-ENOMEM); goto out_err; } #endif + /* allocate dentry private data. We free it in ->d_release */ err = new_dentry_private_data(dentry); if (err) { ret = ERR_PTR(err); goto out; } - ret = __sharefs_lookup(dentry, flags, &lower_parent_path, &lower_path); + ret = __sharefs_lookup(dentry, flags, &lower_parent_path); if (IS_ERR(ret)) { sharefs_err("sharefs_lookup error!"); goto out; @@ -330,7 +327,6 @@ struct dentry *sharefs_lookup(struct inode *dir, struct dentry *dentry, fsstack_copy_attr_atime(d_inode(parent), sharefs_lower_inode(d_inode(parent))); fixup_perm_from_level(d_inode(parent), dentry); - sharefs_put_lower_path(dentry, &lower_path); out: #ifdef CONFIG_SHAREFS_SUPPORT_OVERRIDE sharefs_revert_fsids(saved_cred); diff --git a/fs/sharefs/sharefs.h b/fs/sharefs/sharefs.h index 7b5540b09d33b03e08fc8cc4300360a99d9d08fb..143f047b235f340d0d8122e34ec3b15b9d5ddf6f 100644 --- a/fs/sharefs/sharefs.h +++ b/fs/sharefs/sharefs.h @@ -91,9 +91,6 @@ extern int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt, struct path *path); extern int sharefs_parse_options(struct sharefs_sb_info *sbi, const char *data); -extern int sharefs_get_lower_inode(struct dentry *d, struct inode **lower_path); -extern int sharefs_get_lower_path(struct dentry *d, struct path *lower_path, - bool try_to_create); /* * inode to private data @@ -157,18 +154,16 @@ static inline void pathcpy(struct path *dst, const struct path *src) dst->dentry = src->dentry; dst->mnt = src->mnt; } - -static inline void sharefs_get_lower_root_path(const struct dentry *dent, - struct path *lower_root_path) +/* Returns struct path. Caller must path_put it. */ +static inline void sharefs_get_lower_path(const struct dentry *dent, + struct path *lower_path) { - struct super_block *sb = dent->d_sb; - - spin_lock(&SHAREFS_D(sb->s_root)->lock); - pathcpy(lower_root_path, &SHAREFS_D(sb->s_root)->lower_path); - path_get(lower_root_path); - spin_unlock(&SHAREFS_D(sb->s_root)->lock); + spin_lock(&SHAREFS_D(dent)->lock); + pathcpy(lower_path, &SHAREFS_D(dent)->lower_path); + path_get(lower_path); + spin_unlock(&SHAREFS_D(dent)->lock); + return; } - static inline void sharefs_put_lower_path(const struct dentry *dent, struct path *lower_path) { diff --git a/fs/sharefs/super.c b/fs/sharefs/super.c index b7cd611adcfde79945a73364c3ad8809c7dcf38c..bbe65944647fd611c2aea433b7228f03eeab8e89 100644 --- a/fs/sharefs/super.c +++ b/fs/sharefs/super.c @@ -99,9 +99,7 @@ static int sharefs_statfs(struct dentry *dentry, struct kstatfs *buf) int err; struct path lower_path; - err = sharefs_get_lower_path(dentry, &lower_path, 0); - if (err) - return err; + sharefs_get_lower_path(dentry, &lower_path); err = vfs_statfs(&lower_path, buf); sharefs_put_lower_path(dentry, &lower_path); @@ -119,13 +117,17 @@ static int sharefs_statfs(struct dentry *dentry, struct kstatfs *buf) */ static void sharefs_evict_inode(struct inode *inode) { + struct inode *lower_inode; + truncate_inode_pages(&inode->i_data, 0); clear_inode(inode); /* * Decrement a reference to a lower_inode, which was incremented * by our read_inode when it was created initially. */ + lower_inode = sharefs_lower_inode(inode); sharefs_set_lower_inode(inode, NULL); + iput(lower_inode); } void __sharefs_log(const char *level, const bool ratelimited, diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index 6ccc465aa4265d3e021c3a1546a5b05a82ad46a9..7275725123d7d72a518f0e748ae43687538b6458 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -1228,6 +1228,8 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry, dir_ui->ui_size = dir->i_size; mutex_unlock(&dir_ui->ui_mutex); out_inode: + /* Free inode->i_link before inode is marked as bad. */ + fscrypt_free_inode(inode); make_bad_inode(inode); iput(inode); out_fname: diff --git a/fs/verity/enable.c b/fs/verity/enable.c index ae6ddcea001cda8d6286b29b13e06369a6028cd4..078d2fde21e49fcb74ce29a426f342d34d990219 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -281,6 +281,11 @@ int fsverity_enable_with_descriptor(struct file *filp, struct fsverity_info *vi; int err; + if (vops == NULL) { + fsverity_err(inode, "current filesystem doesn't support fs-verity."); + return -ENOTTY; + } + /* Prepare the Merkle tree parameters */ err = fsverity_init_merkle_tree_params(¶ms, inode, desc->hash_algorithm, @@ -564,6 +569,8 @@ static int code_sign_init_descriptor(struct inode *inode, desc->data_size = cpu_to_le64(arg->data_size); desc->tree_offset = cpu_to_le64(arg->tree_offset); desc->cs_version = arg->cs_version; + desc->pgtypeinfo_size = cpu_to_le32(arg->pgtypeinfo_size); + desc->pgtypeinfo_off = cpu_to_le64(arg->pgtypeinfo_off); /* Get root hash if a Merkle tree carried in file */ if (!IS_INSIDE_TREE(desc)) @@ -605,9 +612,6 @@ int fsverity_ioctl_enable_code_sign(struct file *filp, const void __user *uarg) if (arg.version != 1) return -EINVAL; - if (arg.cs_version != 1) - return -EINVAL; - if (arg.__reserved1 || memchr_inv(arg.__reserved2, 0, sizeof(arg.__reserved2))) return -EINVAL; @@ -626,6 +630,9 @@ int fsverity_ioctl_enable_code_sign(struct file *filp, const void __user *uarg) if (arg.sig_size > FS_VERITY_MAX_SIGNATURE_SIZE) return -EMSGSIZE; + + if (arg.pgtypeinfo_off > arg.data_size - arg.pgtypeinfo_size) + return -EINVAL; return check_file_and_enable_verity(filp, (struct fsverity_enable_arg *)&arg); } diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 0f5366792d22e1a12c9564ee912dc3c3947c9006..e528ca6641d609f1a358c155e30b7995e5075f1f 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -587,8 +587,7 @@ struct acpi_pci_root { bool acpi_dma_supported(struct acpi_device *adev); enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev); -int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, - u64 *size); +int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map); int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, const u32 *input_id); static inline int acpi_dma_configure(struct device *dev, diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index ec66779cb19342e78f862685814b46dbed484b72..79a97a9c4f69551bfd2e2014a9e19d22f2d18149 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -517,7 +517,14 @@ enum acpi_madt_type { ACPI_MADT_TYPE_GENERIC_MSI_FRAME = 13, ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR = 14, ACPI_MADT_TYPE_GENERIC_TRANSLATOR = 15, - ACPI_MADT_TYPE_RESERVED = 16 /* 16 and greater are reserved */ + ACPI_MADT_TYPE_CORE_PIC = 17, + ACPI_MADT_TYPE_LIO_PIC = 18, + ACPI_MADT_TYPE_HT_PIC = 19, + ACPI_MADT_TYPE_EIO_PIC = 20, + ACPI_MADT_TYPE_MSI_PIC = 21, + ACPI_MADT_TYPE_BIO_PIC = 22, + ACPI_MADT_TYPE_LPC_PIC = 23, + ACPI_MADT_TYPE_RESERVED = 24 /* 24 and greater are reserved */ }; /* @@ -724,6 +731,122 @@ struct acpi_madt_generic_translator { u32 reserved2; }; +/* Values for Version field above */ + +enum acpi_madt_core_pic_version { + ACPI_MADT_CORE_PIC_VERSION_NONE = 0, + ACPI_MADT_CORE_PIC_VERSION_V1 = 1, + ACPI_MADT_CORE_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_lio_pic_version { + ACPI_MADT_LIO_PIC_VERSION_NONE = 0, + ACPI_MADT_LIO_PIC_VERSION_V1 = 1, + ACPI_MADT_LIO_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_eio_pic_version { + ACPI_MADT_EIO_PIC_VERSION_NONE = 0, + ACPI_MADT_EIO_PIC_VERSION_V1 = 1, + ACPI_MADT_EIO_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_ht_pic_version { + ACPI_MADT_HT_PIC_VERSION_NONE = 0, + ACPI_MADT_HT_PIC_VERSION_V1 = 1, + ACPI_MADT_HT_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_bio_pic_version { + ACPI_MADT_BIO_PIC_VERSION_NONE = 0, + ACPI_MADT_BIO_PIC_VERSION_V1 = 1, + ACPI_MADT_BIO_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_msi_pic_version { + ACPI_MADT_MSI_PIC_VERSION_NONE = 0, + ACPI_MADT_MSI_PIC_VERSION_V1 = 1, + ACPI_MADT_MSI_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +enum acpi_madt_lpc_pic_version { + ACPI_MADT_LPC_PIC_VERSION_NONE = 0, + ACPI_MADT_LPC_PIC_VERSION_V1 = 1, + ACPI_MADT_LPC_PIC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; + +/* Core Interrupt Controller */ + +struct acpi_madt_core_pic { + struct acpi_subtable_header header; + u8 version; + u32 processor_id; + u32 core_id; + u32 flags; +}; + +/* Legacy I/O Interrupt Controller */ + +struct acpi_madt_lio_pic { + struct acpi_subtable_header header; + u8 version; + u64 address; + u16 size; + u8 cascade[2]; + u32 cascade_map[2]; +}; + +/* Extend I/O Interrupt Controller */ + +struct acpi_madt_eio_pic { + struct acpi_subtable_header header; + u8 version; + u8 cascade; + u8 node; + u64 node_map; +}; + +/* HT Interrupt Controller */ + +struct acpi_madt_ht_pic { + struct acpi_subtable_header header; + u8 version; + u64 address; + u16 size; + u8 cascade[8]; +}; + +/* Bridge I/O Interrupt Controller */ + +struct acpi_madt_bio_pic { + struct acpi_subtable_header header; + u8 version; + u64 address; + u16 size; + u16 id; + u16 gsi_base; +}; + +/* MSI Interrupt Controller */ + +struct acpi_madt_msi_pic { + struct acpi_subtable_header header; + u8 version; + u64 msg_address; + u32 start; + u32 count; +}; + +/* LPC Interrupt Controller */ + +struct acpi_madt_lpc_pic { + struct acpi_subtable_header header; + u8 version; + u64 address; + u16 size; + u8 cascade; +}; + /* * Common flags fields for MADT subtables */ diff --git a/include/acpi/actypes.h b/include/acpi/actypes.h index 7334037624c5c3e493c21d7ce96dd1c893a38e04..3fb9aee1f716c52134d4c1857038cec1a660367b 100644 --- a/include/acpi/actypes.h +++ b/include/acpi/actypes.h @@ -720,7 +720,8 @@ typedef u32 acpi_event_type; #define ACPI_EVENT_POWER_BUTTON 2 #define ACPI_EVENT_SLEEP_BUTTON 3 #define ACPI_EVENT_RTC 4 -#define ACPI_EVENT_MAX 4 +#define ACPI_EVENT_PCIE_WAKE 5 +#define ACPI_EVENT_MAX 5 #define ACPI_NUM_FIXED_EVENTS ACPI_EVENT_MAX + 1 /* diff --git a/include/acpi/platform/aclinux.h b/include/acpi/platform/aclinux.h index 72f52a1342a0f6d818575d49902c30342d0a368e..49f5560505cb2847594c7d6772642e5e6fecdbd9 100644 --- a/include/acpi/platform/aclinux.h +++ b/include/acpi/platform/aclinux.h @@ -194,7 +194,7 @@ #if defined(__ia64__) || (defined(__x86_64__) && !defined(__ILP32__)) ||\ defined(__aarch64__) || defined(__PPC64__) ||\ - defined(__s390x__) ||\ + defined(__s390x__) || defined(__loongarch__) ||\ (defined(__riscv) && (defined(__LP64__) || defined(_LP64))) #define ACPI_MACHINE_WIDTH 64 #define COMPILER_DEPENDENT_INT64 long diff --git a/include/drm/drm_cache.h b/include/drm/drm_cache.h index e9ad4863d91568b4d95047fc98a53284ef93a96b..9d1c3c8da57000650310e40758860681d4426c8a 100644 --- a/include/drm/drm_cache.h +++ b/include/drm/drm_cache.h @@ -65,6 +65,14 @@ static inline bool drm_arch_can_wc_memory(void) * optimization entirely for ARM and arm64. */ return false; +#elif defined(CONFIG_LOONGARCH) + /* + * LoongArch maintains cache coherency in hardware, but its WUC attribute + * (Weak-ordered UnCached, which is similar to WC) is out of the scope of + * cache coherency machanism. This means WUC can only used for write-only + * memory regions. + */ + return false; #else return true; #endif diff --git a/include/dt-bindings/clock/qcom,videocc-sm8150.h b/include/dt-bindings/clock/qcom,videocc-sm8150.h index e24ee840cfdb800f6289dc2236e24a62c65e640b..c557b78dc572f57fa0fad364484d25a355123886 100644 --- a/include/dt-bindings/clock/qcom,videocc-sm8150.h +++ b/include/dt-bindings/clock/qcom,videocc-sm8150.h @@ -16,6 +16,10 @@ /* VIDEO_CC Resets */ #define VIDEO_CC_MVSC_CORE_CLK_BCR 0 +#define VIDEO_CC_INTERFACE_BCR 1 +#define VIDEO_CC_MVS0_BCR 2 +#define VIDEO_CC_MVS1_BCR 3 +#define VIDEO_CC_MVSC_BCR 4 /* VIDEO_CC GDSCRs */ #define VENUS_GDSC 0 diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 9c184dbceba477a6a1f328418a269dbf4750f965..01e44fde11a09c32d575ad3835b929047423892a 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -105,6 +105,7 @@ enum acpi_irq_model_id { ACPI_IRQ_MODEL_IOSAPIC, ACPI_IRQ_MODEL_PLATFORM, ACPI_IRQ_MODEL_GIC, + ACPI_IRQ_MODEL_LPIC, ACPI_IRQ_MODEL_COUNT }; @@ -248,7 +249,7 @@ void acpi_table_print_madt_entry (struct acpi_subtable_header *madt); /* the following numa functions are architecture-dependent */ void acpi_numa_slit_init (struct acpi_table_slit *slit); -#if defined(CONFIG_X86) || defined(CONFIG_IA64) +#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) void acpi_numa_processor_affinity_init (struct acpi_srat_cpu_affinity *pa); #else static inline void @@ -257,6 +258,12 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) { } void acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa); +#if defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH) +void acpi_arch_dma_setup(struct device *dev); +#else +static inline void acpi_arch_dma_setup(struct device *dev) { } +#endif + #ifdef CONFIG_ARM64 void acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa); #else @@ -909,8 +916,7 @@ static inline enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev) return DEV_DMA_NOT_SUPPORTED; } -static inline int acpi_dma_get_range(struct device *dev, u64 *dma_addr, - u64 *offset, u64 *size) +static inline int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map) { return -ENODEV; } diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index 136dba94c646fcaf3868dd63eeb55102c0f91f2a..5870243835cb5f7c03f8714cf6b5e0b7adac3495 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -35,7 +35,7 @@ struct irq_domain *iort_get_device_domain(struct device *dev, u32 id, void acpi_configure_pmsi_domain(struct device *dev); int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); /* IOMMU interface */ -void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *size); +int iort_dma_get_ranges(struct device *dev, u64 *size); const struct iommu_ops *iort_iommu_configure_id(struct device *dev, const u32 *id_in); int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head); @@ -49,8 +49,8 @@ static inline struct irq_domain *iort_get_device_domain( { return NULL; } static inline void acpi_configure_pmsi_domain(struct device *dev) { } /* IOMMU interface */ -static inline void iort_dma_setup(struct device *dev, u64 *dma_addr, - u64 *size) { } +static inline int iort_dma_get_ranges(struct device *dev, u64 *size) +{ return -ENODEV; } static inline const struct iommu_ops *iort_iommu_configure_id( struct device *dev, const u32 *id_in) { return NULL; } diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 4feaf6f92ce9ce5b51419e2c4cfb012c85459858..ba280d26380d1915f226dd39ed489dd3c0d9e166 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -82,7 +82,11 @@ struct bpf_map_ops { /* funcs called by prog_array and perf_event_array map */ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, int fd); - void (*map_fd_put_ptr)(void *ptr); + /* If need_defer is true, the implementation should guarantee that + * the to-be-put element is still alive before the bpf program, which + * may manipulate it, exists. + */ + void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer); int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); u32 (*map_fd_sys_lookup_elem)(void *ptr); void (*map_seq_show_elem)(struct bpf_map *map, void *key, diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index aa8cbf8829145698ec47afbec5e583d146f02ae1..6fa85be64b89699f0de24c02b9f6b5314e4f8edf 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -350,7 +350,7 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, const char *parent_name, const struct clk_hw *parent_hw, const struct clk_parent_data *parent_data, unsigned long flags, unsigned long fixed_rate, unsigned long fixed_accuracy, - unsigned long clk_fixed_flags); + unsigned long clk_fixed_flags, bool devm); struct clk *clk_register_fixed_rate(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned long fixed_rate); @@ -365,7 +365,20 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, */ #define clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \ - NULL, (flags), (fixed_rate), 0, 0) + NULL, (flags), (fixed_rate), 0, 0, false) + +/** + * devm_clk_hw_register_fixed_rate - register fixed-rate clock with the clock + * framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_name: name of clock's parent + * @flags: framework-specific flags + * @fixed_rate: non-adjustable clock rate + */ +#define devm_clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate) \ + __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \ + NULL, (flags), (fixed_rate), 0, 0, true) /** * clk_hw_register_fixed_rate_parent_hw - register fixed-rate clock with * the clock framework @@ -378,7 +391,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, #define clk_hw_register_fixed_rate_parent_hw(dev, name, parent_hw, flags, \ fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw), \ - NULL, (flags), (fixed_rate), 0, 0) + NULL, (flags), (fixed_rate), 0, 0, false) /** * clk_hw_register_fixed_rate_parent_data - register fixed-rate clock with * the clock framework @@ -392,7 +405,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), (flags), (fixed_rate), 0, \ - 0) + 0, false) /** * clk_hw_register_fixed_rate_with_accuracy - register fixed-rate clock with * the clock framework @@ -408,7 +421,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, fixed_accuracy) \ __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), \ NULL, NULL, (flags), (fixed_rate), \ - (fixed_accuracy), 0) + (fixed_accuracy), 0, false) /** * clk_hw_register_fixed_rate_with_accuracy_parent_hw - register fixed-rate * clock with the clock framework @@ -421,9 +434,9 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, */ #define clk_hw_register_fixed_rate_with_accuracy_parent_hw(dev, name, \ parent_hw, flags, fixed_rate, fixed_accuracy) \ - __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw) \ - NULL, NULL, (flags), (fixed_rate), \ - (fixed_accuracy), 0) + __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw), \ + NULL, (flags), (fixed_rate), \ + (fixed_accuracy), 0, false) /** * clk_hw_register_fixed_rate_with_accuracy_parent_data - register fixed-rate * clock with the clock framework @@ -438,7 +451,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, parent_data, flags, fixed_rate, fixed_accuracy) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), NULL, (flags), \ - (fixed_rate), (fixed_accuracy), 0) + (fixed_rate), (fixed_accuracy), 0, false) /** * clk_hw_register_fixed_rate_parent_accuracy - register fixed-rate clock with * the clock framework @@ -452,7 +465,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, flags, fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), (flags), (fixed_rate), 0, \ - CLK_FIXED_RATE_PARENT_ACCURACY) + CLK_FIXED_RATE_PARENT_ACCURACY, false) void clk_unregister_fixed_rate(struct clk *clk); void clk_hw_unregister_fixed_rate(struct clk_hw *hw); diff --git a/include/linux/code_sign.h b/include/linux/code_sign.h index f1d9ab91aa81c0089c9a4d0d4fef374ac6c04aa0..0e4f5574228875be23cc1e6c75add82a5cd64844 100644 --- a/include/linux/code_sign.h +++ b/include/linux/code_sign.h @@ -23,9 +23,10 @@ struct code_sign_descriptor { __u8 root_hash[64]; /* Merkle tree root hash */ __u8 salt[32]; /* salt prepended to each hashed block */ __u32 flags; - __u32 __reserved1; /* must be 0 */ + __u32 pgtypeinfo_size; /* size of page type info (in number of btis) */ __u64 tree_offset; /* merkle tree offset in file */ - __u8 __reserved2[127]; /* must be 0's */ + __u64 pgtypeinfo_off; /* offset of page type info */ + __u8 __reserved2[119]; /* must be 0's */ __u8 cs_version; /* code sign version */ __u8 signature[]; /* optional PKCS#7 signature */ }; diff --git a/include/linux/compat.h b/include/linux/compat.h index 14d514233e1d4007230a83529bd61ce8da1334fd..8dffffe846ce54f8b9d2219ab19ad29afc301600 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -527,7 +527,7 @@ asmlinkage long compat_sys_fstatfs(unsigned int fd, asmlinkage long compat_sys_fstatfs64(unsigned int fd, compat_size_t sz, struct compat_statfs64 __user *buf); asmlinkage long compat_sys_truncate(const char __user *, compat_off_t); -asmlinkage long compat_sys_ftruncate(unsigned int, compat_ulong_t); +asmlinkage long compat_sys_ftruncate(unsigned int, compat_off_t); /* No generic prototype for truncate64, ftruncate64, fallocate */ asmlinkage long compat_sys_openat(int dfd, const char __user *filename, int flags, umode_t mode); diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 475d0a3ce059e9c8cde8042c841e341d0e0774cb..5bfca2f2112c57eaa848f0f70bc4dda241905926 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -113,6 +113,20 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, * The __COUNTER__ based labels are a hack to make each instance of the macros * unique, to convince GCC not to merge duplicate inline asm statements. */ +#ifdef __loongarch__ +#define annotate_reachable() ({ \ + asm volatile("%0:\n\t" \ + ".pushsection .discard.reachable\n\t" \ + ".long %0b - .\n\t" \ + ".popsection\n\t" : : "i" (__COUNTER__)); \ +}) +#define annotate_unreachable() ({ \ + asm volatile("%0:\n\t" \ + ".pushsection .discard.unreachable\n\t" \ + ".long %0b - .\n\t" \ + ".popsection\n\t" : : "i" (__COUNTER__)); \ +}) +#else #define annotate_reachable() ({ \ asm volatile("%c0:\n\t" \ ".pushsection .discard.reachable\n\t" \ @@ -125,6 +139,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, ".long %c0b - .\n\t" \ ".popsection\n\t" : : "i" (__COUNTER__)); \ }) +#endif #define ASM_UNREACHABLE \ "999:\n\t" \ ".pushsection .discard.unreachable\n\t" \ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 088c835ac68e9b6f71fde40d689fcf0d99670fd6..80c351d0d66ce53e6094e4747d37b2ce032570d1 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -108,6 +108,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_BCM2836_STARTING, CPUHP_AP_IRQ_MIPS_GIC_STARTING, CPUHP_AP_IRQ_RISCV_STARTING, + CPUHP_AP_IRQ_LOONGARCH_STARTING, CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, CPUHP_AP_ARM_MVEBU_COHERENCY, CPUHP_AP_MICROCODE_LOADER, diff --git a/include/linux/decompress/mm.h b/include/linux/decompress/mm.h index 868e9eacd69e5e2f81804766b4d61d9b7eb21f87..024aefd05df2f90503c9eb648fc6bf8138465b0c 100644 --- a/include/linux/decompress/mm.h +++ b/include/linux/decompress/mm.h @@ -31,7 +31,7 @@ STATIC_RW_DATA unsigned long malloc_ptr; STATIC_RW_DATA int malloc_count; -static void *malloc(int size) +static void *malloc(size_t size) { void *p; diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 22e3bfebdc34adb84335a21c651681ea7bdb062a..ba1d93c662c6c20d5b6db7a44aa85b41f90be359 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -185,10 +185,10 @@ static inline int dma_declare_coherent_memory(struct device *dev, return -ENOSYS; } -#define dma_release_coherent_memory(dev) (0) #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0) #define dma_release_from_dev_coherent(dev, order, vaddr) (0) #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0) +static inline void dma_release_coherent_memory(struct device *dev) { } static inline void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size, dma_addr_t *dma_handle) diff --git a/include/linux/efi.h b/include/linux/efi.h index 084990390420984f753328fd5b8fad20dcf0d33b..ff717832b34522e40159e0cd576a8b549726efd9 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -363,6 +363,7 @@ void efi_native_runtime_setup(void); #define LINUX_EFI_MEMRESERVE_TABLE_GUID EFI_GUID(0x888eb0c6, 0x8ede, 0x4ff5, 0xa8, 0xf0, 0x9a, 0xee, 0x5c, 0xb9, 0x77, 0xc2) #define LINUX_EFI_INITRD_MEDIA_GUID EFI_GUID(0x5568e427, 0x68fc, 0x4f3d, 0xac, 0x74, 0xca, 0x55, 0x52, 0x31, 0xcc, 0x68) #define LINUX_EFI_MOK_VARIABLE_TABLE_GUID EFI_GUID(0xc451ed2b, 0x9694, 0x45d3, 0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89) +#define LINUX_EFI_BOOT_MEMMAP_GUID EFI_GUID(0x800f683f, 0xd08b, 0x423a, 0xa2, 0x93, 0x96, 0x5c, 0x3c, 0x6f, 0xe2, 0xb4) /* OEM GUIDs */ #define DELLEMC_EFI_RCI2_TABLE_GUID EFI_GUID(0x2d9f28a2, 0xa886, 0x456a, 0x97, 0xa8, 0xf1, 0x1e, 0xf2, 0x4f, 0xf4, 0x55) @@ -456,6 +457,15 @@ typedef union { efi_system_table_32_t mixed_mode; } efi_system_table_t; +struct efi_boot_memmap { + unsigned long map_size; + unsigned long desc_size; + u32 desc_ver; + unsigned long map_key; + unsigned long buff_size; + efi_memory_desc_t map[]; +}; + /* * Architecture independent structure for describing a memory map for the * benefit of efi_memmap_init_early(), and for passing context between @@ -1278,4 +1288,9 @@ static inline struct efi_mokvar_table_entry *efi_mokvar_entry_find( } #endif +struct linux_efi_initrd { + unsigned long base; + unsigned long size; +}; + #endif /* _LINUX_EFI_H */ diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index f1a99d3e5570712f4a48b3984e9b178208a460de..4f4a4f46724f751058a7083432cc0c7905204487 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h @@ -80,7 +80,7 @@ struct dentry; /* * The caller must ensure that fd table isn't shared or hold rcu or file lock */ -static inline struct file *__fcheck_files(struct files_struct *files, unsigned int fd) +static inline struct file *files_lookup_fd_raw(struct files_struct *files, unsigned int fd) { struct fdtable *fdt = rcu_dereference_raw(files->fdt); @@ -91,12 +91,19 @@ static inline struct file *__fcheck_files(struct files_struct *files, unsigned i return NULL; } +static inline struct file *files_lookup_fd_locked(struct files_struct *files, unsigned int fd) +{ + RCU_LOCKDEP_WARN(!lockdep_is_held(&files->file_lock), + "suspicious rcu_dereference_check() usage"); + return files_lookup_fd_raw(files, fd); +} + static inline struct file *fcheck_files(struct files_struct *files, unsigned int fd) { RCU_LOCKDEP_WARN(!rcu_read_lock_held() && !lockdep_is_held(&files->file_lock), "suspicious rcu_dereference_check() usage"); - return __fcheck_files(files, fd); + return files_lookup_fd_raw(files, fd); } /* diff --git a/include/linux/filter.h b/include/linux/filter.h index bc6ce4b202a809820f5446c6b92bd6a4042b2fbb..bfd84789c8261e209a201d514682ebdf9c8d3163 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -823,14 +823,15 @@ bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default) #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0])) -static inline void bpf_prog_lock_ro(struct bpf_prog *fp) +static inline int __must_check bpf_prog_lock_ro(struct bpf_prog *fp) { #ifndef CONFIG_BPF_JIT_ALWAYS_ON if (!fp->jited) { set_vm_flush_reset_perms(fp); - set_memory_ro((unsigned long)fp, fp->pages); + return set_memory_ro((unsigned long)fp, fp->pages); } #endif + return 0; } static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr) diff --git a/include/linux/fs.h b/include/linux/fs.h index b0a52f1c725a43499a08fe16144328c16984dc54..a7cd19c3d3d060ed212c92f0d350aea8dc85e29b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -931,7 +931,6 @@ struct file { */ spinlock_t f_lock; enum rw_hint f_write_hint; - atomic_long_t f_count; unsigned int f_flags; fmode_t f_mode; struct mutex f_pos_lock; @@ -939,6 +938,7 @@ struct file { struct fown_struct f_owner; const struct cred *f_cred; struct file_ra_state f_ra; + atomic_long_t f_count; u64 f_version; #ifdef CONFIG_SECURITY diff --git a/include/linux/iio/adc/adi-axi-adc.h b/include/linux/iio/adc/adi-axi-adc.h index c5d48e1c2d36159449f1a461b413f50492bd504e..77b7f66e6fa80e95d7339a21dfc546d920c69548 100644 --- a/include/linux/iio/adc/adi-axi-adc.h +++ b/include/linux/iio/adc/adi-axi-adc.h @@ -41,6 +41,7 @@ struct adi_axi_adc_chip_info { * @reg_access IIO debugfs_reg_access hook for the client ADC * @read_raw IIO read_raw hook for the client ADC * @write_raw IIO write_raw hook for the client ADC + * @read_avail IIO read_avail hook for the client ADC */ struct adi_axi_adc_conv { const struct adi_axi_adc_chip_info *chip_info; @@ -54,6 +55,9 @@ struct adi_axi_adc_conv { int (*write_raw)(struct adi_axi_adc_conv *conv, struct iio_chan_spec const *chan, int val, int val2, long mask); + int (*read_avail)(struct adi_axi_adc_conv *conv, + struct iio_chan_spec const *chan, + const int **val, int *type, int *length, long mask); }; struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev, diff --git a/include/linux/initrd.h b/include/linux/initrd.h index 8db6f8c8030b688f79178b5425d2dc2854a01eb4..07af5ebc1eaa9f19cc1391d5dd52d3f6e44a4e8c 100644 --- a/include/linux/initrd.h +++ b/include/linux/initrd.h @@ -15,6 +15,12 @@ extern int initrd_below_start_ok; extern unsigned long initrd_start, initrd_end; extern void free_initrd_mem(unsigned long, unsigned long); +#ifdef CONFIG_BLK_DEV_INITRD +extern void __init reserve_initrd_mem(void); +#else +static inline void __init reserve_initrd_mem(void) {} +#endif + extern phys_addr_t phys_initrd_start; extern unsigned long phys_initrd_size; diff --git a/include/linux/kasan.h b/include/linux/kasan.h index c0b976dd138b19896bcf9791734694848b6bc795..09471d24b847a3ef55f8c5971a8cf789f5bc4401 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -29,11 +29,13 @@ extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D]; int kasan_populate_early_shadow(const void *shadow_start, const void *shadow_end); +#ifndef kasan_mem_to_shadow static inline void *kasan_mem_to_shadow(const void *addr) { return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + KASAN_SHADOW_OFFSET; } +#endif /* Enable reporting bugs after kasan_disable_current() */ extern void kasan_enable_current(void); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 31d49b26c03323b376a0064e016efc2271df17d6..0b6d0d5dc0455996bef0e79d36cb77906c350041 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -158,6 +158,8 @@ LSM_HOOK(int, 0, file_alloc_security, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) +LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd, + unsigned long arg) LSM_HOOK(int, 0, mmap_addr, unsigned long addr) LSM_HOOK(int, 0, mmap_file, struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 5807bc877b7838ccc6b995872919b41558bcdccb..393a4348a44805f577ad56a972b4fbd0a4ae9a4e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1237,6 +1237,7 @@ static inline unsigned long section_nr_to_pfn(unsigned long sec) #define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK) struct mem_section_usage { + struct rcu_head rcu; #ifdef CONFIG_SPARSEMEM_VMEMMAP DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION); #endif @@ -1402,7 +1403,7 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn) { int idx = subsection_map_index(pfn); - return test_bit(idx, ms->usage->subsection_map); + return test_bit(idx, READ_ONCE(ms->usage)->subsection_map); } #else static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn) @@ -1415,17 +1416,24 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn) static inline int pfn_valid(unsigned long pfn) { struct mem_section *ms; + int ret; if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; - ms = __nr_to_section(pfn_to_section_nr(pfn)); - if (!valid_section(ms)) + ms = __pfn_to_section(pfn); + rcu_read_lock(); + if (!valid_section(ms)) { + rcu_read_unlock(); return 0; + } /* * Traditionally early sections always returned pfn_valid() for * the entire section-sized span. */ - return early_section(ms) || pfn_section_valid(ms, pfn); + ret = early_section(ms) || pfn_section_valid(ms, pfn); + rcu_read_unlock(); + + return ret; } #endif @@ -1433,7 +1441,7 @@ static inline int pfn_in_present_section(unsigned long pfn) { if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; - return present_section(__nr_to_section(pfn_to_section_nr(pfn))); + return present_section(__pfn_to_section(pfn)); } static inline unsigned long next_present_section_nr(unsigned long section_nr) diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index 033ce74f02e812b9707138209dd7c112d6482b1f..3e28f275a889a51d1bc608318552195e7a951e9e 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -58,6 +58,7 @@ extern const struct pci_ecam_ops pci_thunder_ecam_ops; /* Cavium ThunderX 1.x */ extern const struct pci_ecam_ops xgene_v1_pcie_ecam_ops; /* APM X-Gene PCIe v1 */ extern const struct pci_ecam_ops xgene_v2_pcie_ecam_ops; /* APM X-Gene PCIe v2.x */ extern const struct pci_ecam_ops al_pcie_ops; /* Amazon Annapurna Labs PCIe */ +extern const struct pci_ecam_ops loongson_pci_ecam_ops; /* Loongson PCIe */ #endif #if IS_ENABLED(CONFIG_PCI_HOST_COMMON) diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 1a41147b22e8f7bc854b1be03bb9e5fd10b7efe1..564577fa937469bdca6247bc9be14d4babb94577 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -152,6 +152,20 @@ /* Vendors and devices. Sort key: vendor first, device next. */ #define PCI_VENDOR_ID_LOONGSON 0x0014 +#define PCI_DEVICE_ID_LOONGSON_APB 0x7a02 +#define PCI_DEVICE_ID_LOONGSON_GMAC 0x7a03 +#define PCI_DEVICE_ID_LOONGSON_GNET 0x7a13 +#define PCI_DEVICE_ID_LOONGSON_DC1 0x7a06 +#define PCI_DEVICE_ID_LOONGSON_DC2 0x7a36 +#define PCI_DEVICE_ID_LOONGSON_HDA 0x7a07 +#define PCI_DEVICE_ID_LOONGSON_GPU 0x7a15 +#define PCI_DEVICE_ID_LOONGSON_AHCI 0x7a08 +#define PCI_DEVICE_ID_LOONGSON_EHCI 0x7a14 +#define PCI_DEVICE_ID_LOONGSON_OHCI 0x7a24 +#define PCI_DEVICE_ID_LOONGSON_LPC 0x7a0c +#define PCI_DEVICE_ID_LOONGSON_DMA 0x7a0f +#define PCI_DEVICE_ID_LOONGSON_IOMMU 0x7a1f +#define PCI_DEVICE_ID_LOONGSON_HDMI 0x7a37 #define PCI_VENDOR_ID_TTTECH 0x0357 #define PCI_DEVICE_ID_TTTECH_MC322 0x000a diff --git a/include/linux/pe.h b/include/linux/pe.h index daf09ffffe38992658a7ec25a5c15a6b376de389..f4bb0b6a416d539f81d333c2fff706be69c74286 100644 --- a/include/linux/pe.h +++ b/include/linux/pe.h @@ -65,6 +65,7 @@ #define IMAGE_FILE_MACHINE_SH5 0x01a8 #define IMAGE_FILE_MACHINE_THUMB 0x01c2 #define IMAGE_FILE_MACHINE_WCEMIPSV2 0x0169 +#define IMAGE_FILE_MACHINE_LOONGARCH 0x6264 /* flags */ #define IMAGE_FILE_RELOCS_STRIPPED 0x0001 diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 154e954b711dbf16fe122c27ab79ddc89148e6ea..393f30470c1375775f67016a6afa9a7ce14b4b86 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -116,6 +116,8 @@ extern const struct raid6_calls raid6_vpermxor1; extern const struct raid6_calls raid6_vpermxor2; extern const struct raid6_calls raid6_vpermxor4; extern const struct raid6_calls raid6_vpermxor8; +extern const struct raid6_calls raid6_lsx; +extern const struct raid6_calls raid6_lasx; struct raid6_recov_calls { void (*data2)(int, size_t, int, int, void **); @@ -131,6 +133,8 @@ extern const struct raid6_recov_calls raid6_recov_avx2; extern const struct raid6_recov_calls raid6_recov_avx512; extern const struct raid6_recov_calls raid6_recov_s390xc; extern const struct raid6_recov_calls raid6_recov_neon; +extern const struct raid6_recov_calls raid6_recov_lsx; +extern const struct raid6_recov_calls raid6_recov_lasx; extern const struct raid6_calls raid6_neonx1; extern const struct raid6_calls raid6_neonx2; diff --git a/include/linux/security.h b/include/linux/security.h index f940ab809ad1b3795c2d362d8de2e26785f8a062..12e80c5531b9934bde174e5bd856dfd325c0cee3 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -368,6 +368,8 @@ int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); void security_file_free(struct file *file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int security_file_ioctl_compat(struct file *file, unsigned int cmd, + unsigned long arg); int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags); int security_mmap_addr(unsigned long addr); @@ -925,6 +927,13 @@ static inline int security_file_ioctl(struct file *file, unsigned int cmd, return 0; } +static inline int security_file_ioctl_compat(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + return 0; +} + static inline int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags) { diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 8f5ed0beeea4a9d5afd366ca1d7e6887c8448fb6..85fe78e14e18e61fcd3419837c5de0691fda4aea 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -171,6 +171,7 @@ struct spi_device { #define SPI_MODE_1 (0|SPI_CPHA) #define SPI_MODE_2 (SPI_CPOL|0) #define SPI_MODE_3 (SPI_CPOL|SPI_CPHA) +#define SPI_MODE_X_MASK (SPI_CPOL|SPI_CPHA) #define SPI_CS_HIGH 0x04 /* chipselect active high? */ #define SPI_LSB_FIRST 0x08 /* per-word bits-on-wire */ #define SPI_3WIRE 0x10 /* SI/SO signals shared */ diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index a058c96cf21386ca1d0b5e5280db2101c3e13c43..aa8c1fdbb94dd7f2d4546ccd36fc5a555712dc25 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -119,6 +119,7 @@ struct open_how; #define __TYPE_IS_LL(t) (__TYPE_AS(t, 0LL) || __TYPE_AS(t, 0ULL)) #define __SC_LONG(t, a) __typeof(__builtin_choose_expr(__TYPE_IS_LL(t), 0LL, 0L)) a #define __SC_CAST(t, a) (__force t) a +#define __SC_TYPE(t, a) t #define __SC_ARGS(t, a) a #define __SC_TEST(t, a) (void)BUILD_BUG_ON_ZERO(!__TYPE_IS_LL(t) && sizeof(t) > sizeof(long)) @@ -444,7 +445,7 @@ asmlinkage long sys_fstatfs(unsigned int fd, struct statfs __user *buf); asmlinkage long sys_fstatfs64(unsigned int fd, size_t sz, struct statfs64 __user *buf); asmlinkage long sys_truncate(const char __user *path, long length); -asmlinkage long sys_ftruncate(unsigned int fd, unsigned long length); +asmlinkage long sys_ftruncate(unsigned int fd, off_t length); #if BITS_PER_LONG == 32 asmlinkage long sys_truncate64(const char __user *path, loff_t length); asmlinkage long sys_ftruncate64(unsigned int fd, loff_t length); diff --git a/include/linux/udp.h b/include/linux/udp.h index ae58ff3b6b5b8eb04cca4daabd76b7786d836347..a220880019d6b39e409528889dec0dec5a0a85ff 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -131,6 +131,24 @@ static inline void udp_cmsg_recv(struct msghdr *msg, struct sock *sk, } } +DECLARE_STATIC_KEY_FALSE(udp_encap_needed_key); +#if IS_ENABLED(CONFIG_IPV6) +DECLARE_STATIC_KEY_FALSE(udpv6_encap_needed_key); +#endif + +static inline bool udp_encap_needed(void) +{ + if (static_branch_unlikely(&udp_encap_needed_key)) + return true; + +#if IS_ENABLED(CONFIG_IPV6) + if (static_branch_unlikely(&udpv6_encap_needed_key)) + return true; +#endif + + return false; +} + static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb) { if (!skb_is_gso(skb)) @@ -142,6 +160,16 @@ static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb) if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST && !udp_sk(sk)->accept_udp_fraglist) return true; + /* GSO packets lacking the SKB_GSO_UDP_TUNNEL/_CSUM bits might still + * land in a tunnel as the socket check in udp_gro_receive cannot be + * foolproof. + */ + if (udp_encap_needed() && + READ_ONCE(udp_sk(sk)->encap_rcv) && + !(skb_shinfo(skb)->gso_type & + (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM))) + return true; + return false; } diff --git a/include/linux/units.h b/include/linux/units.h index 3457179f7116a48b5f129b1687c2278b24728938..b61e3f6d509911f465efb88c701583f5b750f25e 100644 --- a/include/linux/units.h +++ b/include/linux/units.h @@ -20,9 +20,13 @@ #define PICO 1000000000000ULL #define FEMTO 1000000000000000ULL -#define MILLIWATT_PER_WATT 1000L -#define MICROWATT_PER_MILLIWATT 1000L -#define MICROWATT_PER_WATT 1000000L +#define HZ_PER_KHZ 1000UL +#define KHZ_PER_MHZ 1000UL +#define HZ_PER_MHZ 1000000UL + +#define MILLIWATT_PER_WATT 1000UL +#define MICROWATT_PER_MILLIWATT 1000UL +#define MICROWATT_PER_WATT 1000000UL #define ABSOLUTE_ZERO_MILLICELSIUS -273150 diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 76dad53a410ac85a3d4dc965c7010865cdf63361..0fd47f2f39eb07a09766d7c54416b0b2f498539b 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -112,6 +112,11 @@ extern void *__vmalloc_node_range(unsigned long size, unsigned long align, void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller); +extern void *__vmalloc_array(size_t n, size_t size, gfp_t flags); +extern void *vmalloc_array(size_t n, size_t size); +extern void *__vcalloc(size_t n, size_t size, gfp_t flags); +extern void *vcalloc(size_t n, size_t size); + extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h index 14efa0ded75dd93d6f63735a95feaf72de6fdef7..3c3508ecea6a157dd8b277acd53c9eaa14ac8ce2 100644 --- a/include/net/dst_metadata.h +++ b/include/net/dst_metadata.h @@ -124,7 +124,6 @@ static inline struct metadata_dst *tun_dst_unclone(struct sk_buff *skb) memcpy(&new_md->u.tun_info, &md_dst->u.tun_info, sizeof(struct ip_tunnel_info) + md_size); skb_dst_drop(skb); - dst_hold(&new_md->dst); skb_dst_set(skb, &new_md->dst); return new_md; } diff --git a/include/net/llc_pdu.h b/include/net/llc_pdu.h index 49aa79c7b278a540ea98361316ddf47b412f5302..581cd37aa98b72cb1a8182db69f24ef6aa9f95bd 100644 --- a/include/net/llc_pdu.h +++ b/include/net/llc_pdu.h @@ -262,8 +262,7 @@ static inline void llc_pdu_header_init(struct sk_buff *skb, u8 type, */ static inline void llc_pdu_decode_sa(struct sk_buff *skb, u8 *sa) { - if (skb->protocol == htons(ETH_P_802_2)) - memcpy(sa, eth_hdr(skb)->h_source, ETH_ALEN); + memcpy(sa, eth_hdr(skb)->h_source, ETH_ALEN); } /** @@ -275,8 +274,7 @@ static inline void llc_pdu_decode_sa(struct sk_buff *skb, u8 *sa) */ static inline void llc_pdu_decode_da(struct sk_buff *skb, u8 *da) { - if (skb->protocol == htons(ETH_P_802_2)) - memcpy(da, eth_hdr(skb)->h_dest, ETH_ALEN); + memcpy(da, eth_hdr(skb)->h_dest, ETH_ALEN); } /** diff --git a/include/net/lowpower_protocol.h b/include/net/lowpower_protocol.h index afe525e121928f8f50bf3af2022481b84ad9588b..656b0741fc04d6d9c992b90628b079aa527bb573 100644 --- a/include/net/lowpower_protocol.h +++ b/include/net/lowpower_protocol.h @@ -26,5 +26,6 @@ int tcp_ack_num(struct sock *sk); bool netfilter_bypass_enable(struct net *net, struct sk_buff *skb, int (*fun)(struct net *, struct sock *, struct sk_buff *), int *ret); +bool dpa_uid_match(uid_t kuid); #endif /* CONFIG_LOWPOWER_PROTOCOL */ #endif /* __LOWPOWER_PROTOCOL_H */ \ No newline at end of file diff --git a/include/net/xdp.h b/include/net/xdp.h index 9dab2bc6f187bee09e0ed05c5de6623168b3b667..9e6c10b323b8eee48dc32228de01eecf92b4239f 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -218,6 +218,9 @@ bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, enum xdp_mem_type type, void *allocator); void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); +int xdp_reg_mem_model(struct xdp_mem_info *mem, + enum xdp_mem_type type, void *allocator); +void xdp_unreg_mem_model(struct xdp_mem_info *mem); /* Drivers not supporting XDP metadata can use this helper, which * rejects any room expansion for metadata as a result. diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index cb4e8e6e86a90b7c10a749740ee11cef0005e5d3..01f047415ab0a519d797e25401108abc9bb86fcc 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -434,6 +434,8 @@ enum { #define AUDIT_ARCH_UNICORE (EM_UNICORE|__AUDIT_ARCH_LE) #define AUDIT_ARCH_X86_64 (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) #define AUDIT_ARCH_XTENSA (EM_XTENSA) +#define AUDIT_ARCH_LOONGARCH32 (EM_LOONGARCH|__AUDIT_ARCH_LE) +#define AUDIT_ARCH_LOONGARCH64 (EM_LOONGARCH|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) #define AUDIT_PERM_EXEC 1 #define AUDIT_PERM_WRITE 2 diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index e65300d63d7c4e06ca8548fc400e880bc134c372..cf4431b748c232e4c9ba410890fad3b2026ffd2a 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -576,6 +576,9 @@ struct btrfs_ioctl_clone_range_args { */ #define BTRFS_DEFRAG_RANGE_COMPRESS 1 #define BTRFS_DEFRAG_RANGE_START_IO 2 +#define BTRFS_DEFRAG_RANGE_FLAGS_SUPP (BTRFS_DEFRAG_RANGE_COMPRESS | \ + BTRFS_DEFRAG_RANGE_START_IO) + struct btrfs_ioctl_defrag_range_args { /* start of the defrag operation */ __u64 start; diff --git a/include/uapi/linux/elf-em.h b/include/uapi/linux/elf-em.h index f47e853546faa94bf520b7ce6d76c1edaec800c6..ef38c2bc5ab7a58ba03d309654051b25a2ac2933 100644 --- a/include/uapi/linux/elf-em.h +++ b/include/uapi/linux/elf-em.h @@ -51,6 +51,7 @@ #define EM_RISCV 243 /* RISC-V */ #define EM_BPF 247 /* Linux BPF - in-kernel virtual machine */ #define EM_CSKY 252 /* C-SKY */ +#define EM_LOONGARCH 258 /* LoongArch */ #define EM_FRV 0x5441 /* Fujitsu FR-V */ /* diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index 2ae08e605c3c7fc20a15d23d45f5521decf1e94f..85590f879d53122e65e96b618e162ee43867b381 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -434,6 +434,11 @@ typedef struct elf64_shdr { #define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */ #define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode */ #define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */ +#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */ +#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */ +#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */ +#define NT_LOONGARCH_LASX 0xa03 /* LoongArch Loongson Advanced SIMD Extension registers */ +#define NT_LOONGARCH_LBT 0xa04 /* LoongArch Loongson Binary Translation registers */ /* Note types with note name "GNU" */ #define NT_GNU_PROPERTY_TYPE_0 5 diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index ae44df1b1d1763a85761f4002b69ed8b9cde3113..b5a820ad13c06ed361510ccdba6e215224802d02 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -46,7 +46,10 @@ struct code_sign_enable_arg { __u32 sig_size; __u32 __reserved1; __u64 sig_ptr; - __u64 __reserved2[7]; + __u64 __reserved2[5]; + __u32 __reserved3; + __u32 pgtypeinfo_size; + __u64 pgtypeinfo_off; __u64 tree_offset; __u64 root_hash_ptr; __u64 data_size; diff --git a/include/uapi/linux/kexec.h b/include/uapi/linux/kexec.h index 05669c87a0afb93a4863186f3789d730b4c2c18e..d55db902fe6e224fa77860b84206eb3d5c5b4976 100644 --- a/include/uapi/linux/kexec.h +++ b/include/uapi/linux/kexec.h @@ -42,6 +42,7 @@ #define KEXEC_ARCH_MIPS_LE (10 << 16) #define KEXEC_ARCH_MIPS ( 8 << 16) #define KEXEC_ARCH_AARCH64 (183 << 16) +#define KEXEC_ARCH_LOONGARCH (258 << 16) /* The artificial cap on the number of segments passed to kexec_load. */ #define KEXEC_SEGMENT_MAX 16 diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ca41220b40b8b25c8f1044496f8024458768868b..bfe1e6a5cdb0182e3f2c1f3e9b3ed9c63016da22 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -250,6 +250,7 @@ struct kvm_hyperv_exit { #define KVM_EXIT_ARM_NISV 28 #define KVM_EXIT_X86_RDMSR 29 #define KVM_EXIT_X86_WRMSR 30 +#define KVM_EXIT_LOONGARCH_IOCSR 36 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -319,6 +320,13 @@ struct kvm_run { __u32 len; __u8 is_write; } mmio; + /* KVM_EXIT_LOONGARCH_IOCSR */ + struct { + __u64 phys_addr; + __u8 data[8]; + __u32 len; + __u8 is_write; + } iocsr_io; /* KVM_EXIT_HYPERCALL */ struct { __u64 nr; @@ -695,6 +703,16 @@ struct kvm_s390_irq_state { __u32 reserved[4]; /* will stay unused for compatibility reasons */ }; +struct kvm_loongarch_vcpu_state { + __u8 online_vcpus; + __u8 is_migrate; + __u32 cpu_freq; + __u32 count_ctl; + __u64 irq_pending; + __u64 irq_clear; + __u64 core_ext_ioisr[4]; +}; + /* for KVM_SET_GUEST_DEBUG */ #define KVM_GUESTDBG_ENABLE 0x00000001 @@ -1053,6 +1071,9 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_X86_USER_SPACE_MSR 188 #define KVM_CAP_X86_MSR_FILTER 189 #define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190 +#define KVM_CAP_LOONGARCH_FPU 191 +#define KVM_CAP_LOONGARCH_LSX 192 +#define KVM_CAP_LOONGARCH_VZ 193 #ifdef KVM_CAP_IRQ_ROUTING @@ -1200,6 +1221,7 @@ struct kvm_dirty_tlb { #define KVM_REG_ARM64 0x6000000000000000ULL #define KVM_REG_MIPS 0x7000000000000000ULL #define KVM_REG_RISCV 0x8000000000000000ULL +#define KVM_REG_LOONGARCH 0x9000000000000000ULL #define KVM_REG_SIZE_SHIFT 52 #define KVM_REG_SIZE_MASK 0x00f0000000000000ULL @@ -1557,6 +1579,13 @@ struct kvm_pv_cmd { /* Available with KVM_CAP_X86_MSR_FILTER */ #define KVM_X86_SET_MSR_FILTER _IOW(KVMIO, 0xc6, struct kvm_msr_filter) +#define KVM_LOONGARCH_GET_VCPU_STATE _IOR(KVMIO, 0xd0, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_SET_VCPU_STATE _IOW(KVMIO, 0xd1, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_GET_CPUCFG _IOR(KVMIO, 0xd2, struct kvm_cpucfg) +#define KVM_LOONGARCH_GET_IOCSR _IOR(KVMIO, 0xd3, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_IOCSR _IOW(KVMIO, 0xd4, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_CPUCFG _IOR(KVMIO, 0xd5, struct kvm_cpucfg) + /* Secure Encrypted Virtualization command */ enum sev_cmd_id { /* Guest initialization commands */ diff --git a/init/Kconfig b/init/Kconfig index 70bd7f69076ab1f3310ba3abe2d1a8f73b3c6c55..8cfe761ffdf68c6cd185ec8a6cd04c445c05d143 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1296,6 +1296,17 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. +config RT_SOFTINT_OPTIMIZATION + bool "Improve RT scheduling during long softint execution" + depends on (ARM64 || LOONGARCH) + depends on SMP + default n + help + Enable an optimization which tries to avoid placing RT tasks on CPUs + occupied by nonpreemptible tasks, such as a long softint, or CPUs + which may soon block preemptions, such as a CPU running a ksoftirq + thread which handles slow softints. + config SYSFS_DEPRECATED bool "Enable deprecated sysfs features to support old userspace tools" depends on SYSFS diff --git a/init/do_mounts.c b/init/do_mounts.c index b5f9604d0c98a22307397836f224945c6e762778..8ef154fb418c158b318a99e85236424d18c81547 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -649,7 +649,10 @@ struct file_system_type rootfs_fs_type = { void __init init_rootfs(void) { - if (IS_ENABLED(CONFIG_TMPFS) && !saved_root_name[0] && - (!root_fs_names || strstr(root_fs_names, "tmpfs"))) - is_tmpfs = true; + if (IS_ENABLED(CONFIG_TMPFS)) { + if (!saved_root_name[0] && !root_fs_names) + is_tmpfs = true; + else if (root_fs_names && !!strstr(root_fs_names, "tmpfs")) + is_tmpfs = true; + } } diff --git a/init/initramfs.c b/init/initramfs.c index 55b74d7e526075d94145af4430037b52413722a2..f75c89e9d602d8be9bc87b4ca4997339ccf521c8 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -535,6 +535,51 @@ extern unsigned long __initramfs_size; #include #include +void __init reserve_initrd_mem(void) +{ + phys_addr_t start; + unsigned long size; + + /* Ignore the virtul address computed during device tree parsing */ + initrd_start = initrd_end = 0; + + if (!phys_initrd_size) + return; + /* + * Round the memory region to page boundaries as per free_initrd_mem() + * This allows us to detect whether the pages overlapping the initrd + * are in use, but more importantly, reserves the entire set of pages + * as we don't want these pages allocated for other purposes. + */ + start = round_down(phys_initrd_start, PAGE_SIZE); + size = phys_initrd_size + (phys_initrd_start - start); + size = round_up(size, PAGE_SIZE); + + if (!memblock_is_region_memory(start, size)) { + pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region", + (u64)start, size); + goto disable; + } + + if (memblock_is_region_reserved(start, size)) { + pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n", + (u64)start, size); + goto disable; + } + + memblock_reserve(start, size); + /* Now convert initrd to virtual addresses */ + initrd_start = (unsigned long)__va(phys_initrd_start); + initrd_end = initrd_start + phys_initrd_size; + initrd_below_start_ok = 1; + + return; +disable: + pr_cont(" - disabling initrd\n"); + initrd_start = 0; + initrd_end = 0; +} + void __weak __init free_initrd_mem(unsigned long start, unsigned long end) { #ifdef CONFIG_ARCH_KEEP_MEMBLOCK diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index ea9cf4cac47b8308a0f952fc1014d610284bee82..907553ad639fce6c8beabecb447645b32ae27fc6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3485,14 +3485,17 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) struct iovec *iov = iorw->fast_iov; int ret; + iorw->bytes_done = 0; + iorw->free_iovec = NULL; + ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); if (unlikely(ret < 0)) return ret; - iorw->bytes_done = 0; - iorw->free_iovec = iov; - if (iov) + if (iov) { + iorw->free_iovec = iov; req->flags |= REQ_F_NEED_CLEANUP; + } iov_iter_save_state(&iorw->iter, &iorw->iter_state); return 0; } diff --git a/kernel/audit.c b/kernel/audit.c index aeec86ed470883f28acde2f3b73011252f4d4d66..2ab04e0a74418992077afab09c1f390a361d8939 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -490,15 +490,19 @@ static void auditd_conn_free(struct rcu_head *rcu) * @pid: auditd PID * @portid: auditd netlink portid * @net: auditd network namespace pointer + * @skb: the netlink command from the audit daemon + * @ack: netlink ack flag, cleared if ack'd here * * Description: * This function will obtain and drop network namespace references as * necessary. Returns zero on success, negative values on failure. */ -static int auditd_set(struct pid *pid, u32 portid, struct net *net) +static int auditd_set(struct pid *pid, u32 portid, struct net *net, + struct sk_buff *skb, bool *ack) { unsigned long flags; struct auditd_connection *ac_old, *ac_new; + struct nlmsghdr *nlh; if (!pid || !net) return -EINVAL; @@ -510,6 +514,13 @@ static int auditd_set(struct pid *pid, u32 portid, struct net *net) ac_new->portid = portid; ac_new->net = get_net(net); + /* send the ack now to avoid a race with the queue backlog */ + if (*ack) { + nlh = nlmsg_hdr(skb); + netlink_ack(skb, nlh, 0, NULL); + *ack = false; + } + spin_lock_irqsave(&auditd_conn_lock, flags); ac_old = rcu_dereference_protected(auditd_conn, lockdep_is_held(&auditd_conn_lock)); @@ -1203,7 +1214,8 @@ static int audit_replace(struct pid *pid) return auditd_send_unicast_skb(skb); } -static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) +static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh, + bool *ack) { u32 seq; void *data; @@ -1296,7 +1308,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) /* register a new auditd connection */ err = auditd_set(req_pid, NETLINK_CB(skb).portid, - sock_net(NETLINK_CB(skb).sk)); + sock_net(NETLINK_CB(skb).sk), + skb, ack); if (audit_enabled != AUDIT_OFF) audit_log_config_change("audit_pid", new_pid, @@ -1541,9 +1554,10 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) * Parse the provided skb and deal with any messages that may be present, * malformed skbs are discarded. */ -static void audit_receive(struct sk_buff *skb) +static void audit_receive(struct sk_buff *skb) { struct nlmsghdr *nlh; + bool ack; /* * len MUST be signed for nlmsg_next to be able to dec it below 0 * if the nlmsg_len was not aligned @@ -1556,9 +1570,12 @@ static void audit_receive(struct sk_buff *skb) audit_ctl_lock(); while (nlmsg_ok(nlh, len)) { - err = audit_receive_msg(skb, nlh); - /* if err or if this message says it wants a response */ - if (err || (nlh->nlmsg_flags & NLM_F_ACK)) + ack = nlh->nlmsg_flags & NLM_F_ACK; + err = audit_receive_msg(skb, nlh, &ack); + + /* send an ack if the user asked for one and audit_receive_msg + * didn't already do it, or if there was an error. */ + if (ack || err) netlink_ack(skb, nlh, err, NULL); nlh = nlmsg_next(nlh, &len); diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index f241bda2679d43d765c08eec6a6287bd6a8e4fa5..5102338129d5f6e5a777aed4bf9cce313816a1df 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -764,7 +764,7 @@ int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, } if (old_ptr) - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, true); return 0; } @@ -787,7 +787,7 @@ static int fd_array_map_delete_elem(struct bpf_map *map, void *key) } if (old_ptr) { - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, true); return 0; } else { return -ENOENT; @@ -811,8 +811,9 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, return prog; } -static void prog_fd_array_put_ptr(void *ptr) +static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_prog is freed after one RCU or tasks trace grace period */ bpf_prog_put(ptr); } @@ -1139,8 +1140,9 @@ static void *perf_event_fd_array_get_ptr(struct bpf_map *map, return ee; } -static void perf_event_fd_array_put_ptr(void *ptr) +static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_perf_event is freed after one RCU grace period */ bpf_event_entry_free_rcu(ptr); } @@ -1195,7 +1197,7 @@ static void *cgroup_fd_array_get_ptr(struct bpf_map *map, return cgroup_get_from_fd(fd); } -static void cgroup_fd_array_put_ptr(void *ptr) +static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { /* cgroup_put free cgrp after a rcu grace period */ cgroup_put(ptr); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 33ea6ab12f47c72d5062f743cf6dca8f82078ca0..6fd3f8d0dda33ed615de50ac362a67ffa9ee2df3 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1882,7 +1882,9 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) } finalize: - bpf_prog_lock_ro(fp); + *err = bpf_prog_lock_ro(fp); + if (*err) + return fp; /* The tail call compatibility check can only be done at * this late stage as we need to determine, if we deal diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index d05614a2bdc7583c8a445652e0063d92c6a25a5d..72bc5f5752543707b9f1416c8f8a26eb5d01a10c 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -790,7 +790,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) if (map->ops->map_fd_put_ptr) { ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, true); } } @@ -2027,7 +2027,7 @@ static void fd_htab_map_free(struct bpf_map *map) hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { void *ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); } } @@ -2068,7 +2068,7 @@ int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, ret = htab_map_update_elem(map, key, &ptr, map_flags); if (ret) - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); return ret; } diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 39ab0b68cade574acb7a8226566e30aa91307ee1..0cf4cb685810557f78e29394dbbea5eae35319c9 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -100,7 +100,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return inner_map; } -void bpf_map_fd_put_ptr(void *ptr) +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { /* ptr->ops->map_free() has to go through one * rcu grace period by itself. diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h index bcb7534afb3c0d073222a20bc7d38538e986ee27..7d61602354de8074126944aeea4bb1d116b939e1 100644 --- a/kernel/bpf/map_in_map.h +++ b/kernel/bpf/map_in_map.h @@ -13,7 +13,7 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd); void bpf_map_meta_free(struct bpf_map *map_meta); void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file, int ufd); -void bpf_map_fd_put_ptr(void *ptr); +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer); u32 bpf_map_fd_sys_lookup_elem(void *ptr); #endif diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index d6fbe17432ae5f74872c2d2f88394063197277b1..d21d4ba2eb391c11af779cdb7a7dc4dadaa3f7d7 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -44,6 +44,7 @@ struct bpf_ringbuf { */ unsigned long consumer_pos __aligned(PAGE_SIZE); unsigned long producer_pos __aligned(PAGE_SIZE); + unsigned long pending_pos; char data[] __aligned(PAGE_SIZE); }; @@ -145,6 +146,7 @@ static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node) rb->mask = data_sz - 1; rb->consumer_pos = 0; rb->producer_pos = 0; + rb->pending_pos = 0; return rb; } @@ -323,9 +325,9 @@ bpf_ringbuf_restore_from_rec(struct bpf_ringbuf_hdr *hdr) static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) { - unsigned long cons_pos, prod_pos, new_prod_pos, flags; - u32 len, pg_off; + unsigned long cons_pos, prod_pos, new_prod_pos, pend_pos, flags; struct bpf_ringbuf_hdr *hdr; + u32 len, pg_off, tmp_size, hdr_len; if (unlikely(size > RINGBUF_MAX_RECORD_SZ)) return NULL; @@ -343,13 +345,29 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) spin_lock_irqsave(&rb->spinlock, flags); } + pend_pos = rb->pending_pos; prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - /* check for out of ringbuf space by ensuring producer position - * doesn't advance more than (ringbuf_size - 1) ahead + while (pend_pos < prod_pos) { + hdr = (void *)rb->data + (pend_pos & rb->mask); + hdr_len = READ_ONCE(hdr->len); + if (hdr_len & BPF_RINGBUF_BUSY_BIT) + break; + tmp_size = hdr_len & ~BPF_RINGBUF_DISCARD_BIT; + tmp_size = round_up(tmp_size + BPF_RINGBUF_HDR_SZ, 8); + pend_pos += tmp_size; + } + rb->pending_pos = pend_pos; + + /* check for out of ringbuf space: + * - by ensuring producer position doesn't advance more than + * (ringbuf_size - 1) ahead + * - by ensuring oldest not yet committed record until newest + * record does not span more than (ringbuf_size - 1) */ - if (new_prod_pos - cons_pos > rb->mask) { + if (new_prod_pos - cons_pos > rb->mask || + new_prod_pos - pend_pos > rb->mask) { spin_unlock_irqrestore(&rb->spinlock, flags); return NULL; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ec424fba002013336709394df86973e604a324ef..07df0f27060af1de8dc02590c256f42d44c6ea98 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1285,6 +1285,9 @@ int generic_map_delete_batch(struct bpf_map *map, if (!max_count) return 0; + if (put_user(0, &uattr->batch.count)) + return -EFAULT; + key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); if (!key) return -ENOMEM; @@ -1343,6 +1346,9 @@ int generic_map_update_batch(struct bpf_map *map, if (!max_count) return 0; + if (put_user(0, &uattr->batch.count)) + return -EFAULT; + key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN); if (!key) return -ENOMEM; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6fc79a7babd5b9e4f138648c5e8bf69b3a1c7262..5a14aa62967c378e4a675820372a17b6081c0f31 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11528,11 +11528,15 @@ static int jit_subprogs(struct bpf_verifier_env *env) /* finally lock prog and jit images for all functions and * populate kallsysm */ - for (i = 0; i < env->subprog_cnt; i++) { - bpf_prog_lock_ro(func[i]); - bpf_prog_kallsyms_add(func[i]); + for (i = 1; i < env->subprog_cnt; i++) { + err = bpf_prog_lock_ro(func[i]); + if (err) + goto out_free; } + for (i = 1; i < env->subprog_cnt; i++) + bpf_prog_kallsyms_add(func[i]); + /* Last step: make now unused interpreter insns from main * prog consistent for later dump requests, so they can * later look the same as if they were interpreted only. diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index 4e09fab52faf50beed7f4af7de696d4689c16d40..c27b3dfa192107b0b609e32c3ebfab834efbca04 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c @@ -1364,8 +1364,6 @@ static int kdb_local(kdb_reason_t reason, int error, struct pt_regs *regs, /* PROMPT can only be set if we have MEM_READ permission. */ snprintf(kdb_prompt_str, CMD_BUFLEN, kdbgetenv("PROMPT"), raw_smp_processor_id()); - if (defcmd_in_progress) - strncat(kdb_prompt_str, "[defcmd]", CMD_BUFLEN); /* * Fetch command from keyboard diff --git a/kernel/events/core.c b/kernel/events/core.c index afedd008e0afdddbb3efde95e92864aeb31350e1..ab5b75f3b88663e19d019175c1204d6a6c72c4fa 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -10855,9 +10855,32 @@ static DEVICE_ATTR_RW(perf_event_mux_interval_ms); static struct attribute *pmu_dev_attrs[] = { &dev_attr_type.attr, &dev_attr_perf_event_mux_interval_ms.attr, + &dev_attr_nr_addr_filters.attr, + NULL, +}; + +static umode_t pmu_dev_is_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct pmu *pmu = dev_get_drvdata(dev); + + if (!pmu->nr_addr_filters) + return 0; + + return a->mode; + + return 0; +} + +static struct attribute_group pmu_dev_attr_group = { + .is_visible = pmu_dev_is_visible, + .attrs = pmu_dev_attrs, +}; + +static const struct attribute_group *pmu_dev_groups[] = { + &pmu_dev_attr_group, NULL, }; -ATTRIBUTE_GROUPS(pmu_dev); static int pmu_bus_running; static struct bus_type pmu_bus = { @@ -10893,18 +10916,11 @@ static int pmu_dev_alloc(struct pmu *pmu) if (ret) goto free_dev; - /* For PMUs with address filters, throw in an extra attribute: */ - if (pmu->nr_addr_filters) - ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters); - - if (ret) - goto del_dev; - - if (pmu->attr_update) + if (pmu->attr_update) { ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update); - - if (ret) - goto del_dev; + if (ret) + goto del_dev; + } out: return ret; diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 4457f3e966d0ef6386d3636116c952d56337a5ae..548cee01144c9b2dafd4761de242d27bf72a07ad 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -301,6 +301,12 @@ struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, return domain; } +/* + * Return Val: + * = 0: Success; + * > 0: The modified nvec; + * < 0: Error code. + */ int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg) { @@ -407,8 +413,10 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, bool can_reserve; ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); - if (ret) + if (ret < 0) return ret; + if (ret > 0) + nvec = ret; for_each_msi_entry(desc, dev) { ops->set_desc(&arg, desc); diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 1d3975b36f9f79389ac4e2ccd7bad5a6007d1889..dba6541c0fc3c421c9208015d3ad22e69dbf51df 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -545,17 +545,15 @@ static void do_unoptimize_kprobes(void) /* See comment in do_optimize_kprobes() */ lockdep_assert_cpus_held(); - /* Unoptimization must be done anytime */ - if (list_empty(&unoptimizing_list)) - return; + if (!list_empty(&unoptimizing_list)) + arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); - arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); - /* Loop free_list for disarming */ + /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */ list_for_each_entry_safe(op, tmp, &freeing_list, list) { /* Switching from detour code to origin */ op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; - /* Disarm probes if marked disabled */ - if (kprobe_disabled(&op->kp)) + /* Disarm probes if marked disabled and not gone */ + if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp)) arch_disarm_kprobe(&op->kp); if (kprobe_unused(&op->kp)) { /* @@ -784,14 +782,13 @@ static void kill_optimized_kprobe(struct kprobe *p) op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; if (kprobe_unused(p)) { - /* Enqueue if it is unused */ - list_add(&op->list, &freeing_list); /* - * Remove unused probes from the hash list. After waiting - * for synchronization, this probe is reclaimed. - * (reclaiming is done by do_free_cleaned_kprobes().) + * Unused kprobe is on unoptimizing or freeing list. We move it + * to freeing_list and let the kprobe_optimizer() remove it from + * the kprobe hash list and free it. */ - hlist_del_rcu(&op->kp.hlist); + if (optprobe_queued_unopt(op)) + list_move(&op->list, &freeing_list); } /* Don't touch the code, because it is already freed. */ diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index f5faf935c2d8f8674e583d002eef831aaeba8b8a..a7cda7ede045411bbc08dc37466d846de2b726ef 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -198,7 +198,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, int i, cnt, ret; char sym_objname[MODULE_NAME_LEN]; char sym_name[KSYM_NAME_LEN]; - Elf_Rela *relas; + Elf_Rela *relas, *last = NULL; Elf_Sym *sym; unsigned long sympos, addr; bool sym_vmlinux; @@ -219,6 +219,10 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, relas = (Elf_Rela *) relasec->sh_addr; /* For each rela in this klp relocation section */ for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) { + if (last && relas[i].r_offset == last->r_offset) + continue; + + last = &relas[i]; sym = (Elf_Sym *)sechdrs[symndx].sh_addr + ELF_R_SYM(relas[i].r_info); if (sym->st_shndx != SHN_LIVEPATCH) { pr_err("symbol %s is not marked as a livepatch symbol\n", diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 25e7cb96bb8840ed7f4977d78774056fff8f55db..b288aba8040c2c15a6456ce8b13c818cd9c332c9 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -603,11 +603,11 @@ static int crc32_threadfn(void *data) unsigned i; while (1) { - wait_event(d->go, atomic_read(&d->ready) || + wait_event(d->go, atomic_read_acquire(&d->ready) || kthread_should_stop()); if (kthread_should_stop()) { d->thr = NULL; - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); break; } @@ -616,7 +616,7 @@ static int crc32_threadfn(void *data) for (i = 0; i < d->run_threads; i++) *d->crc32 = crc32_le(*d->crc32, d->unc[i], *d->unc_len[i]); - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); } return 0; @@ -646,12 +646,12 @@ static int lzo_compress_threadfn(void *data) struct cmp_data *d = data; while (1) { - wait_event(d->go, atomic_read(&d->ready) || + wait_event(d->go, atomic_read_acquire(&d->ready) || kthread_should_stop()); if (kthread_should_stop()) { d->thr = NULL; d->ret = -1; - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); break; } @@ -660,7 +660,7 @@ static int lzo_compress_threadfn(void *data) d->ret = lzo1x_1_compress(d->unc, d->unc_len, d->cmp + LZO_HEADER, &d->cmp_len, d->wrk); - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); } return 0; @@ -798,7 +798,7 @@ static int save_image_lzo(struct swap_map_handle *handle, data[thr].unc_len = off; - atomic_set(&data[thr].ready, 1); + atomic_set_release(&data[thr].ready, 1); wake_up(&data[thr].go); } @@ -806,12 +806,12 @@ static int save_image_lzo(struct swap_map_handle *handle, break; crc->run_threads = thr; - atomic_set(&crc->ready, 1); + atomic_set_release(&crc->ready, 1); wake_up(&crc->go); for (run_threads = thr, thr = 0; thr < run_threads; thr++) { wait_event(data[thr].done, - atomic_read(&data[thr].stop)); + atomic_read_acquire(&data[thr].stop)); atomic_set(&data[thr].stop, 0); ret = data[thr].ret; @@ -850,7 +850,7 @@ static int save_image_lzo(struct swap_map_handle *handle, } } - wait_event(crc->done, atomic_read(&crc->stop)); + wait_event(crc->done, atomic_read_acquire(&crc->stop)); atomic_set(&crc->stop, 0); } @@ -1132,12 +1132,12 @@ static int lzo_decompress_threadfn(void *data) struct dec_data *d = data; while (1) { - wait_event(d->go, atomic_read(&d->ready) || + wait_event(d->go, atomic_read_acquire(&d->ready) || kthread_should_stop()); if (kthread_should_stop()) { d->thr = NULL; d->ret = -1; - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); break; } @@ -1150,7 +1150,7 @@ static int lzo_decompress_threadfn(void *data) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); - atomic_set(&d->stop, 1); + atomic_set_release(&d->stop, 1); wake_up(&d->done); } return 0; @@ -1338,7 +1338,7 @@ static int load_image_lzo(struct swap_map_handle *handle, } if (crc->run_threads) { - wait_event(crc->done, atomic_read(&crc->stop)); + wait_event(crc->done, atomic_read_acquire(&crc->stop)); atomic_set(&crc->stop, 0); crc->run_threads = 0; } @@ -1374,7 +1374,7 @@ static int load_image_lzo(struct swap_map_handle *handle, pg = 0; } - atomic_set(&data[thr].ready, 1); + atomic_set_release(&data[thr].ready, 1); wake_up(&data[thr].go); } @@ -1393,7 +1393,7 @@ static int load_image_lzo(struct swap_map_handle *handle, for (run_threads = thr, thr = 0; thr < run_threads; thr++) { wait_event(data[thr].done, - atomic_read(&data[thr].stop)); + atomic_read_acquire(&data[thr].stop)); atomic_set(&data[thr].stop, 0); ret = data[thr].ret; @@ -1424,7 +1424,7 @@ static int load_image_lzo(struct swap_map_handle *handle, ret = snapshot_write_next(snapshot); if (ret <= 0) { crc->run_threads = thr + 1; - atomic_set(&crc->ready, 1); + atomic_set_release(&crc->ready, 1); wake_up(&crc->go); goto out_finish; } @@ -1432,13 +1432,13 @@ static int load_image_lzo(struct swap_map_handle *handle, } crc->run_threads = thr; - atomic_set(&crc->ready, 1); + atomic_set_release(&crc->ready, 1); wake_up(&crc->go); } out_finish: if (crc->run_threads) { - wait_event(crc->done, atomic_read(&crc->stop)); + wait_event(crc->done, atomic_read_acquire(&crc->stop)); atomic_set(&crc->stop, 0); } stop = ktime_get(); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index fc79b04b594707d89f9ddfa0aea1882ad1c7315a..d1693c26958fcfeeda6b7918949ff77377dca583 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1439,13 +1439,23 @@ void tick_setup_sched_timer(void) void tick_cancel_sched_timer(int cpu) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); + ktime_t idle_sleeptime, iowait_sleeptime; + unsigned long idle_calls, idle_sleeps; # ifdef CONFIG_HIGH_RES_TIMERS if (ts->sched_timer.base) hrtimer_cancel(&ts->sched_timer); # endif + idle_sleeptime = ts->idle_sleeptime; + iowait_sleeptime = ts->iowait_sleeptime; + idle_calls = ts->idle_calls; + idle_sleeps = ts->idle_sleeps; memset(ts, 0, sizeof(*ts)); + ts->idle_sleeptime = idle_sleeptime; + ts->iowait_sleeptime = iowait_sleeptime; + ts->idle_calls = idle_calls; + ts->idle_sleeps = idle_sleeps; } #endif diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index 51a9d1185033b981d4227a765ed00c31d9cda4ae..d47641f9740bcc537b414761662f7e44a1580964 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -574,7 +574,12 @@ __tracing_map_insert(struct tracing_map *map, void *key, bool lookup_only) } memcpy(elt->key, key, map->key_size); - entry->val = elt; + /* + * Ensure the initialization is visible and + * publish the elt. + */ + smp_wmb(); + WRITE_ONCE(entry->val, elt); atomic64_inc(&map->hits); return entry->val; diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 542a9c18398e158ccec96f5adf952bd23737fd16..6e5363f91aed9f67776bbbbf2e7042bd12f05650 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -118,6 +118,8 @@ endchoice config KASAN_STACK_ENABLE bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST + depends on KASAN_GENERIC || KASAN_SW_TAGS + default y if (CC_IS_GCC && !LOONGARCH) help The LLVM stack address sanitizer has a know problem that causes excessive stack usage in a lot of functions, see diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 4dd9283f6fea0fbda9570135246f93bd34c2ce04..b055741a5a4ddd7bf62bb5fe21e53a81772a22cc 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -612,9 +612,8 @@ static void debug_objects_fill_pool(void) static void __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack) { - enum debug_obj_state state; + struct debug_obj *obj, o; struct debug_bucket *db; - struct debug_obj *obj; unsigned long flags; debug_objects_fill_pool(); @@ -635,24 +634,18 @@ __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack case ODEBUG_STATE_INIT: case ODEBUG_STATE_INACTIVE: obj->state = ODEBUG_STATE_INIT; - break; - - case ODEBUG_STATE_ACTIVE: - state = obj->state; - raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "init"); - debug_object_fixup(descr->fixup_init, addr, state); - return; - - case ODEBUG_STATE_DESTROYED: raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "init"); return; default: break; } + o = *obj; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(&o, "init"); + + if (o.state == ODEBUG_STATE_ACTIVE) + debug_object_fixup(descr->fixup_init, addr, o.state); } /** @@ -693,11 +686,9 @@ EXPORT_SYMBOL_GPL(debug_object_init_on_stack); int debug_object_activate(void *addr, const struct debug_obj_descr *descr) { struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr }; - enum debug_obj_state state; struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; - int ret; if (!debug_objects_enabled) return 0; @@ -709,49 +700,38 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr) raw_spin_lock_irqsave(&db->lock, flags); obj = lookup_object_or_alloc(addr, db, descr, false, true); - if (likely(!IS_ERR_OR_NULL(obj))) { - bool print_object = false; - + if (unlikely(!obj)) { + raw_spin_unlock_irqrestore(&db->lock, flags); + debug_objects_oom(); + return 0; + } else if (likely(!IS_ERR(obj))) { switch (obj->state) { - case ODEBUG_STATE_INIT: - case ODEBUG_STATE_INACTIVE: - obj->state = ODEBUG_STATE_ACTIVE; - ret = 0; - break; - case ODEBUG_STATE_ACTIVE: - state = obj->state; - raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "activate"); - ret = debug_object_fixup(descr->fixup_activate, addr, state); - return ret ? 0 : -EINVAL; - case ODEBUG_STATE_DESTROYED: - print_object = true; - ret = -EINVAL; + o = *obj; break; + case ODEBUG_STATE_INIT: + case ODEBUG_STATE_INACTIVE: + obj->state = ODEBUG_STATE_ACTIVE; + fallthrough; default: - ret = 0; - break; + raw_spin_unlock_irqrestore(&db->lock, flags); + return 0; } - raw_spin_unlock_irqrestore(&db->lock, flags); - if (print_object) - debug_print_object(obj, "activate"); - return ret; } raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(&o, "activate"); - /* If NULL the allocation has hit OOM */ - if (!obj) { - debug_objects_oom(); - return 0; + switch (o.state) { + case ODEBUG_STATE_ACTIVE: + case ODEBUG_STATE_NOTAVAILABLE: + if (debug_object_fixup(descr->fixup_activate, addr, o.state)) + return 0; + fallthrough; + default: + return -EINVAL; } - - /* Object is neither static nor tracked. It's not initialized */ - debug_print_object(&o, "activate"); - ret = debug_object_fixup(descr->fixup_activate, addr, ODEBUG_STATE_NOTAVAILABLE); - return ret ? 0 : -EINVAL; } EXPORT_SYMBOL_GPL(debug_object_activate); @@ -762,10 +742,10 @@ EXPORT_SYMBOL_GPL(debug_object_activate); */ void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr) { + struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr }; struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; - bool print_object = false; if (!debug_objects_enabled) return; @@ -777,33 +757,24 @@ void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr) obj = lookup_object(addr, db); if (obj) { switch (obj->state) { + case ODEBUG_STATE_DESTROYED: + break; case ODEBUG_STATE_INIT: case ODEBUG_STATE_INACTIVE: case ODEBUG_STATE_ACTIVE: - if (!obj->astate) - obj->state = ODEBUG_STATE_INACTIVE; - else - print_object = true; - break; - - case ODEBUG_STATE_DESTROYED: - print_object = true; - break; + if (obj->astate) + break; + obj->state = ODEBUG_STATE_INACTIVE; + fallthrough; default: - break; + raw_spin_unlock_irqrestore(&db->lock, flags); + return; } + o = *obj; } raw_spin_unlock_irqrestore(&db->lock, flags); - if (!obj) { - struct debug_obj o = { .object = addr, - .state = ODEBUG_STATE_NOTAVAILABLE, - .descr = descr }; - - debug_print_object(&o, "deactivate"); - } else if (print_object) { - debug_print_object(obj, "deactivate"); - } + debug_print_object(&o, "deactivate"); } EXPORT_SYMBOL_GPL(debug_object_deactivate); @@ -814,11 +785,9 @@ EXPORT_SYMBOL_GPL(debug_object_deactivate); */ void debug_object_destroy(void *addr, const struct debug_obj_descr *descr) { - enum debug_obj_state state; + struct debug_obj *obj, o; struct debug_bucket *db; - struct debug_obj *obj; unsigned long flags; - bool print_object = false; if (!debug_objects_enabled) return; @@ -828,32 +797,31 @@ void debug_object_destroy(void *addr, const struct debug_obj_descr *descr) raw_spin_lock_irqsave(&db->lock, flags); obj = lookup_object(addr, db); - if (!obj) - goto out_unlock; + if (!obj) { + raw_spin_unlock_irqrestore(&db->lock, flags); + return; + } switch (obj->state) { + case ODEBUG_STATE_ACTIVE: + case ODEBUG_STATE_DESTROYED: + break; case ODEBUG_STATE_NONE: case ODEBUG_STATE_INIT: case ODEBUG_STATE_INACTIVE: obj->state = ODEBUG_STATE_DESTROYED; - break; - case ODEBUG_STATE_ACTIVE: - state = obj->state; + fallthrough; + default: raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "destroy"); - debug_object_fixup(descr->fixup_destroy, addr, state); return; - - case ODEBUG_STATE_DESTROYED: - print_object = true; - break; - default: - break; } -out_unlock: + + o = *obj; raw_spin_unlock_irqrestore(&db->lock, flags); - if (print_object) - debug_print_object(obj, "destroy"); + debug_print_object(&o, "destroy"); + + if (o.state == ODEBUG_STATE_ACTIVE) + debug_object_fixup(descr->fixup_destroy, addr, o.state); } EXPORT_SYMBOL_GPL(debug_object_destroy); @@ -864,9 +832,8 @@ EXPORT_SYMBOL_GPL(debug_object_destroy); */ void debug_object_free(void *addr, const struct debug_obj_descr *descr) { - enum debug_obj_state state; + struct debug_obj *obj, o; struct debug_bucket *db; - struct debug_obj *obj; unsigned long flags; if (!debug_objects_enabled) @@ -877,24 +844,26 @@ void debug_object_free(void *addr, const struct debug_obj_descr *descr) raw_spin_lock_irqsave(&db->lock, flags); obj = lookup_object(addr, db); - if (!obj) - goto out_unlock; + if (!obj) { + raw_spin_unlock_irqrestore(&db->lock, flags); + return; + } switch (obj->state) { case ODEBUG_STATE_ACTIVE: - state = obj->state; - raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "free"); - debug_object_fixup(descr->fixup_free, addr, state); - return; + break; default: hlist_del(&obj->node); raw_spin_unlock_irqrestore(&db->lock, flags); free_object(obj); return; } -out_unlock: + + o = *obj; raw_spin_unlock_irqrestore(&db->lock, flags); + debug_print_object(&o, "free"); + + debug_object_fixup(descr->fixup_free, addr, o.state); } EXPORT_SYMBOL_GPL(debug_object_free); @@ -946,10 +915,10 @@ void debug_object_active_state(void *addr, const struct debug_obj_descr *descr, unsigned int expect, unsigned int next) { + struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr }; struct debug_bucket *db; struct debug_obj *obj; unsigned long flags; - bool print_object = false; if (!debug_objects_enabled) return; @@ -962,28 +931,19 @@ debug_object_active_state(void *addr, const struct debug_obj_descr *descr, if (obj) { switch (obj->state) { case ODEBUG_STATE_ACTIVE: - if (obj->astate == expect) - obj->astate = next; - else - print_object = true; - break; - + if (obj->astate != expect) + break; + obj->astate = next; + raw_spin_unlock_irqrestore(&db->lock, flags); + return; default: - print_object = true; break; } + o = *obj; } raw_spin_unlock_irqrestore(&db->lock, flags); - if (!obj) { - struct debug_obj o = { .object = addr, - .state = ODEBUG_STATE_NOTAVAILABLE, - .descr = descr }; - - debug_print_object(&o, "active_state"); - } else if (print_object) { - debug_print_object(obj, "active_state"); - } + debug_print_object(&o, "active_state"); } EXPORT_SYMBOL_GPL(debug_object_active_state); @@ -991,12 +951,10 @@ EXPORT_SYMBOL_GPL(debug_object_active_state); static void __debug_check_no_obj_freed(const void *address, unsigned long size) { unsigned long flags, oaddr, saddr, eaddr, paddr, chunks; - const struct debug_obj_descr *descr; - enum debug_obj_state state; + int cnt, objs_checked = 0; + struct debug_obj *obj, o; struct debug_bucket *db; struct hlist_node *tmp; - struct debug_obj *obj; - int cnt, objs_checked = 0; saddr = (unsigned long) address; eaddr = saddr + size; @@ -1018,12 +976,10 @@ static void __debug_check_no_obj_freed(const void *address, unsigned long size) switch (obj->state) { case ODEBUG_STATE_ACTIVE: - descr = obj->descr; - state = obj->state; + o = *obj; raw_spin_unlock_irqrestore(&db->lock, flags); - debug_print_object(obj, "free"); - debug_object_fixup(descr->fixup_free, - (void *) oaddr, state); + debug_print_object(&o, "free"); + debug_object_fixup(o.descr->fixup_free, (void *)oaddr, o.state); goto repeat; default: hlist_del(&obj->node); diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 10a50c03074ee14b7ed73b7bb8612c340a51d942..685cf3e6771d3f30622c00d76d518b5a8ee57ff9 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -260,7 +260,11 @@ static int ddebug_tokenize(char *buf, char *words[], int maxwords) } else { for (end = buf; *end && !isspace(*end); end++) ; - BUG_ON(end == buf); + if (end == buf) { + pr_err("parse err after word:%d=%s\n", nwords, + nwords ? words[nwords - 1] : ""); + return -EINVAL; + } } /* `buf' is start of word, `end' is one past its end */ diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index c770570bfe4f2d939a1331657f45a43edade9467..e2b1ef565f0d4c825962d0744eb4bb3416bdeabb 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -9,6 +9,7 @@ raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o hostprogs += mktables diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 6d5e5000fdd7967b77d49425a384ba7d7287cb9f..1187431d5cd6808237fb5fa9eaefa8ae1649bf5e 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -75,6 +75,14 @@ const struct raid6_calls * const raid6_algos[] = { &raid6_neonx2, &raid6_neonx1, #endif +#ifdef CONFIG_LOONGARCH +#ifdef CONFIG_CPU_HAS_LASX + &raid6_lasx, +#endif +#ifdef CONFIG_CPU_HAS_LSX + &raid6_lsx, +#endif +#endif #if defined(__ia64__) &raid6_intx32, &raid6_intx16, @@ -105,6 +113,14 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { #endif #if defined(CONFIG_KERNEL_MODE_NEON) &raid6_recov_neon, +#endif +#ifdef CONFIG_LOONGARCH +#ifdef CONFIG_CPU_HAS_LASX + &raid6_recov_lasx, +#endif +#ifdef CONFIG_CPU_HAS_LSX + &raid6_recov_lsx, +#endif #endif &raid6_recov_intx1, NULL diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h new file mode 100644 index 0000000000000000000000000000000000000000..acfc33ce705625058131f4fd54bd4c5a64af0d30 --- /dev/null +++ b/lib/raid6/loongarch.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2023 WANG Xuerui + * + * raid6/loongarch.h + * + * Definitions common to LoongArch RAID-6 code only + */ + +#ifndef _LIB_RAID6_LOONGARCH_H +#define _LIB_RAID6_LOONGARCH_H + +#ifdef __KERNEL__ + +#include +#include + +#else /* for user-space testing */ + +#include + +/* have to supply these defines for glibc 2.37- and musl */ +#ifndef HWCAP_LOONGARCH_LSX +#define HWCAP_LOONGARCH_LSX (1 << 4) +#endif +#ifndef HWCAP_LOONGARCH_LASX +#define HWCAP_LOONGARCH_LASX (1 << 5) +#endif + +#define kernel_fpu_begin() +#define kernel_fpu_end() + +#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) +#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) + +#endif /* __KERNEL__ */ + +#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/loongarch_simd.c b/lib/raid6/loongarch_simd.c new file mode 100644 index 0000000000000000000000000000000000000000..6032b83a427a92a5881a710e14283ee953136359 --- /dev/null +++ b/lib/raid6/loongarch_simd.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * RAID6 syndrome calculations in LoongArch SIMD (LSX & LASX) + * + * Copyright 2023 WANG Xuerui + * + * Based on the generic RAID-6 code (int.uc): + * + * Copyright 2002-2004 H. Peter Anvin + */ + +#include +#include "loongarch.h" + +/* + * The vector algorithms are currently priority 0, which means the generic + * scalar algorithms are not being disabled if vector support is present. + * This is like the similar LoongArch RAID5 XOR code, with the main reason + * repeated here: it cannot be ruled out at this point of time, that some + * future (maybe reduced) models could run the vector algorithms slower than + * the scalar ones, maybe for errata or micro-op reasons. It may be + * appropriate to revisit this after one or two more uarch generations. + */ + +#ifdef CONFIG_CPU_HAS_LSX +#define NSIZE 16 + +static int raid6_has_lsx(void) +{ + return cpu_has_lsx; +} + +static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = disks - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $vr0, $vr1, $vr2, $vr3: wp + * $vr4, $vr5, $vr6, $vr7: wq + * $vr8, $vr9, $vr10, $vr11: wd + * $vr12, $vr13, $vr14, $vr15: w2 + * $vr16, $vr17, $vr18, $vr19: w1 + */ + for (d = 0; d < bytes; d += NSIZE*4) { + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE])); + asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE])); + asm volatile("vori.b $vr4, $vr0, 0"); + asm volatile("vori.b $vr5, $vr1, 0"); + asm volatile("vori.b $vr6, $vr2, 0"); + asm volatile("vori.b $vr7, $vr3, 0"); + for (z = z0-1; z >= 0; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE])); + asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE])); + asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("vxor.v $vr16, $vr16, $vr12"); + asm volatile("vxor.v $vr17, $vr17, $vr13"); + asm volatile("vxor.v $vr18, $vr18, $vr14"); + asm volatile("vxor.v $vr19, $vr19, $vr15"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr8"); + asm volatile("vxor.v $vr5, $vr17, $vr9"); + asm volatile("vxor.v $vr6, $vr18, $vr10"); + asm volatile("vxor.v $vr7, $vr19, $vr11"); + } + /* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */ + asm volatile("vst $vr0, %0" : "=m"(p[d+NSIZE*0])); + asm volatile("vst $vr1, %0" : "=m"(p[d+NSIZE*1])); + asm volatile("vst $vr2, %0" : "=m"(p[d+NSIZE*2])); + asm volatile("vst $vr3, %0" : "=m"(p[d+NSIZE*3])); + /* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */ + asm volatile("vst $vr4, %0" : "=m"(q[d+NSIZE*0])); + asm volatile("vst $vr5, %0" : "=m"(q[d+NSIZE*1])); + asm volatile("vst $vr6, %0" : "=m"(q[d+NSIZE*2])); + asm volatile("vst $vr7, %0" : "=m"(q[d+NSIZE*3])); + } + + kernel_fpu_end(); +} + +static void raid6_lsx_xor_syndrome(int disks, int start, int stop, + size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = stop; /* P/Q right side optimization */ + p = dptr[disks-2]; /* XOR parity */ + q = dptr[disks-1]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $vr0, $vr1, $vr2, $vr3: wp + * $vr4, $vr5, $vr6, $vr7: wq + * $vr8, $vr9, $vr10, $vr11: wd + * $vr12, $vr13, $vr14, $vr15: w2 + * $vr16, $vr17, $vr18, $vr19: w1 + */ + for (d = 0; d < bytes; d += NSIZE*4) { + /* P/Q data pages */ + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE])); + asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE])); + asm volatile("vori.b $vr4, $vr0, 0"); + asm volatile("vori.b $vr5, $vr1, 0"); + asm volatile("vori.b $vr6, $vr2, 0"); + asm volatile("vori.b $vr7, $vr3, 0"); + for (z = z0-1; z >= start; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE])); + asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE])); + asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("vxor.v $vr16, $vr16, $vr12"); + asm volatile("vxor.v $vr17, $vr17, $vr13"); + asm volatile("vxor.v $vr18, $vr18, $vr14"); + asm volatile("vxor.v $vr19, $vr19, $vr15"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr8"); + asm volatile("vxor.v $vr5, $vr17, $vr9"); + asm volatile("vxor.v $vr6, $vr18, $vr10"); + asm volatile("vxor.v $vr7, $vr19, $vr11"); + } + + /* P/Q left side optimization */ + for (z = start-1; z >= 0; z--) { + /* w2$$ = MASK(wq$$); */ + asm volatile("vslti.b $vr12, $vr4, 0"); + asm volatile("vslti.b $vr13, $vr5, 0"); + asm volatile("vslti.b $vr14, $vr6, 0"); + asm volatile("vslti.b $vr15, $vr7, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("vslli.b $vr16, $vr4, 1"); + asm volatile("vslli.b $vr17, $vr5, 1"); + asm volatile("vslli.b $vr18, $vr6, 1"); + asm volatile("vslli.b $vr19, $vr7, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("vandi.b $vr12, $vr12, 0x1d"); + asm volatile("vandi.b $vr13, $vr13, 0x1d"); + asm volatile("vandi.b $vr14, $vr14, 0x1d"); + asm volatile("vandi.b $vr15, $vr15, 0x1d"); + /* wq$$ = w1$$ ^ w2$$; */ + asm volatile("vxor.v $vr4, $vr16, $vr12"); + asm volatile("vxor.v $vr5, $vr17, $vr13"); + asm volatile("vxor.v $vr6, $vr18, $vr14"); + asm volatile("vxor.v $vr7, $vr19, $vr15"); + } + /* + * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$; + * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$; + */ + asm volatile( + "vld $vr20, %0\n\t" + "vld $vr21, %1\n\t" + "vld $vr22, %2\n\t" + "vld $vr23, %3\n\t" + "vld $vr24, %4\n\t" + "vld $vr25, %5\n\t" + "vld $vr26, %6\n\t" + "vld $vr27, %7\n\t" + "vxor.v $vr20, $vr20, $vr0\n\t" + "vxor.v $vr21, $vr21, $vr1\n\t" + "vxor.v $vr22, $vr22, $vr2\n\t" + "vxor.v $vr23, $vr23, $vr3\n\t" + "vxor.v $vr24, $vr24, $vr4\n\t" + "vxor.v $vr25, $vr25, $vr5\n\t" + "vxor.v $vr26, $vr26, $vr6\n\t" + "vxor.v $vr27, $vr27, $vr7\n\t" + "vst $vr20, %0\n\t" + "vst $vr21, %1\n\t" + "vst $vr22, %2\n\t" + "vst $vr23, %3\n\t" + "vst $vr24, %4\n\t" + "vst $vr25, %5\n\t" + "vst $vr26, %6\n\t" + "vst $vr27, %7\n\t" + : "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]), + "+m"(p[d+NSIZE*2]), "+m"(p[d+NSIZE*3]), + "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]), + "+m"(q[d+NSIZE*2]), "+m"(q[d+NSIZE*3]) + ); + } + + kernel_fpu_end(); +} + +const struct raid6_calls raid6_lsx = { + raid6_lsx_gen_syndrome, + raid6_lsx_xor_syndrome, + raid6_has_lsx, + "lsx", +}; + +#undef NSIZE +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +#define NSIZE 32 + +static int raid6_has_lasx(void) +{ + return cpu_has_lasx; +} + +static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = disks - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $xr0, $xr1: wp + * $xr2, $xr3: wq + * $xr4, $xr5: wd + * $xr6, $xr7: w2 + * $xr8, $xr9: w1 + */ + for (d = 0; d < bytes; d += NSIZE*2) { + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("xvori.b $xr2, $xr0, 0"); + asm volatile("xvori.b $xr3, $xr1, 0"); + for (z = z0-1; z >= 0; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("xvxor.v $xr8, $xr8, $xr6"); + asm volatile("xvxor.v $xr9, $xr9, $xr7"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr4"); + asm volatile("xvxor.v $xr3, $xr9, $xr5"); + } + /* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */ + asm volatile("xvst $xr0, %0" : "=m"(p[d+NSIZE*0])); + asm volatile("xvst $xr1, %0" : "=m"(p[d+NSIZE*1])); + /* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */ + asm volatile("xvst $xr2, %0" : "=m"(q[d+NSIZE*0])); + asm volatile("xvst $xr3, %0" : "=m"(q[d+NSIZE*1])); + } + + kernel_fpu_end(); +} + +static void raid6_lasx_xor_syndrome(int disks, int start, int stop, + size_t bytes, void **ptrs) +{ + u8 **dptr = (u8 **)ptrs; + u8 *p, *q; + int d, z, z0; + + z0 = stop; /* P/Q right side optimization */ + p = dptr[disks-2]; /* XOR parity */ + q = dptr[disks-1]; /* RS syndrome */ + + kernel_fpu_begin(); + + /* + * $xr0, $xr1: wp + * $xr2, $xr3: wq + * $xr4, $xr5: wd + * $xr6, $xr7: w2 + * $xr8, $xr9: w1 + */ + for (d = 0; d < bytes; d += NSIZE*2) { + /* P/Q data pages */ + /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */ + asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE])); + asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE])); + asm volatile("xvori.b $xr2, $xr0, 0"); + asm volatile("xvori.b $xr3, $xr1, 0"); + for (z = z0-1; z >= start; z--) { + /* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */ + asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE])); + asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE])); + /* wp$$ ^= wd$$; */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* w1$$ ^= w2$$; */ + asm volatile("xvxor.v $xr8, $xr8, $xr6"); + asm volatile("xvxor.v $xr9, $xr9, $xr7"); + /* wq$$ = w1$$ ^ wd$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr4"); + asm volatile("xvxor.v $xr3, $xr9, $xr5"); + } + + /* P/Q left side optimization */ + for (z = start-1; z >= 0; z--) { + /* w2$$ = MASK(wq$$); */ + asm volatile("xvslti.b $xr6, $xr2, 0"); + asm volatile("xvslti.b $xr7, $xr3, 0"); + /* w1$$ = SHLBYTE(wq$$); */ + asm volatile("xvslli.b $xr8, $xr2, 1"); + asm volatile("xvslli.b $xr9, $xr3, 1"); + /* w2$$ &= NBYTES(0x1d); */ + asm volatile("xvandi.b $xr6, $xr6, 0x1d"); + asm volatile("xvandi.b $xr7, $xr7, 0x1d"); + /* wq$$ = w1$$ ^ w2$$; */ + asm volatile("xvxor.v $xr2, $xr8, $xr6"); + asm volatile("xvxor.v $xr3, $xr9, $xr7"); + } + /* + * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$; + * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$; + */ + asm volatile( + "xvld $xr10, %0\n\t" + "xvld $xr11, %1\n\t" + "xvld $xr12, %2\n\t" + "xvld $xr13, %3\n\t" + "xvxor.v $xr10, $xr10, $xr0\n\t" + "xvxor.v $xr11, $xr11, $xr1\n\t" + "xvxor.v $xr12, $xr12, $xr2\n\t" + "xvxor.v $xr13, $xr13, $xr3\n\t" + "xvst $xr10, %0\n\t" + "xvst $xr11, %1\n\t" + "xvst $xr12, %2\n\t" + "xvst $xr13, %3\n\t" + : "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]), + "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]) + ); + } + + kernel_fpu_end(); +} + +const struct raid6_calls raid6_lasx = { + raid6_lasx_gen_syndrome, + raid6_lasx_xor_syndrome, + raid6_has_lasx, + "lasx", +}; +#undef NSIZE +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid6/recov_loongarch_simd.c new file mode 100644 index 0000000000000000000000000000000000000000..94aeac85e6f75e393363868c6306aff8d4521a5b --- /dev/null +++ b/lib/raid6/recov_loongarch_simd.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RAID6 recovery algorithms in LoongArch SIMD (LSX & LASX) + * + * Copyright (C) 2023 WANG Xuerui + * + * Originally based on recov_avx2.c and recov_ssse3.c: + * + * Copyright (C) 2012 Intel Corporation + * Author: Jim Kukunas + */ + +#include +#include "loongarch.h" + +/* + * Unlike with the syndrome calculation algorithms, there's no boot-time + * selection of recovery algorithms by benchmarking, so we have to specify + * the priorities and hope the future cores will all have decent vector + * support (i.e. no LASX slower than LSX, or even scalar code). + */ + +#ifdef CONFIG_CPU_HAS_LSX +static int raid6_has_lsx(void) +{ + return cpu_has_lsx; +} + +static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, + int failb, void **ptrs) +{ + u8 *p, *q, *dp, *dq; + const u8 *pbmul; /* P multiplier table for B data */ + const u8 *qmul; /* Q multiplier table (for both) */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 2] = dp; + dq = (u8 *)ptrs[failb]; + ptrs[failb] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dp; + ptrs[failb] = dq; + ptrs[disks - 2] = p; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; + + kernel_fpu_begin(); + + /* + * vr20, vr21: qmul + * vr22, vr23: pbmul + */ + asm volatile("vld $vr20, %0" : : "m" (qmul[0])); + asm volatile("vld $vr21, %0" : : "m" (qmul[16])); + asm volatile("vld $vr22, %0" : : "m" (pbmul[0])); + asm volatile("vld $vr23, %0" : : "m" (pbmul[16])); + + while (bytes) { + /* vr4 - vr7: Q */ + asm volatile("vld $vr4, %0" : : "m" (q[0])); + asm volatile("vld $vr5, %0" : : "m" (q[16])); + asm volatile("vld $vr6, %0" : : "m" (q[32])); + asm volatile("vld $vr7, %0" : : "m" (q[48])); + /* vr4 - vr7: Q + Qxy */ + asm volatile("vld $vr8, %0" : : "m" (dq[0])); + asm volatile("vld $vr9, %0" : : "m" (dq[16])); + asm volatile("vld $vr10, %0" : : "m" (dq[32])); + asm volatile("vld $vr11, %0" : : "m" (dq[48])); + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + /* vr0 - vr3: P */ + asm volatile("vld $vr0, %0" : : "m" (p[0])); + asm volatile("vld $vr1, %0" : : "m" (p[16])); + asm volatile("vld $vr2, %0" : : "m" (p[32])); + asm volatile("vld $vr3, %0" : : "m" (p[48])); + /* vr0 - vr3: P + Pxy */ + asm volatile("vld $vr8, %0" : : "m" (dp[0])); + asm volatile("vld $vr9, %0" : : "m" (dp[16])); + asm volatile("vld $vr10, %0" : : "m" (dp[32])); + asm volatile("vld $vr11, %0" : : "m" (dp[48])); + asm volatile("vxor.v $vr0, $vr0, $vr8"); + asm volatile("vxor.v $vr1, $vr1, $vr9"); + asm volatile("vxor.v $vr2, $vr2, $vr10"); + asm volatile("vxor.v $vr3, $vr3, $vr11"); + + /* vr8 - vr11: higher 4 bits of each byte of (Q + Qxy) */ + asm volatile("vsrli.b $vr8, $vr4, 4"); + asm volatile("vsrli.b $vr9, $vr5, 4"); + asm volatile("vsrli.b $vr10, $vr6, 4"); + asm volatile("vsrli.b $vr11, $vr7, 4"); + /* vr4 - vr7: lower 4 bits of each byte of (Q + Qxy) */ + asm volatile("vandi.b $vr4, $vr4, 0x0f"); + asm volatile("vandi.b $vr5, $vr5, 0x0f"); + asm volatile("vandi.b $vr6, $vr6, 0x0f"); + asm volatile("vandi.b $vr7, $vr7, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("vshuf.b $vr4, $vr20, $vr20, $vr4"); + asm volatile("vshuf.b $vr5, $vr20, $vr20, $vr5"); + asm volatile("vshuf.b $vr6, $vr20, $vr20, $vr6"); + asm volatile("vshuf.b $vr7, $vr20, $vr20, $vr7"); + /* lookup from qmul[16] */ + asm volatile("vshuf.b $vr8, $vr21, $vr21, $vr8"); + asm volatile("vshuf.b $vr9, $vr21, $vr21, $vr9"); + asm volatile("vshuf.b $vr10, $vr21, $vr21, $vr10"); + asm volatile("vshuf.b $vr11, $vr21, $vr21, $vr11"); + /* vr16 - vr19: B(Q + Qxy) */ + asm volatile("vxor.v $vr16, $vr8, $vr4"); + asm volatile("vxor.v $vr17, $vr9, $vr5"); + asm volatile("vxor.v $vr18, $vr10, $vr6"); + asm volatile("vxor.v $vr19, $vr11, $vr7"); + + /* vr4 - vr7: higher 4 bits of each byte of (P + Pxy) */ + asm volatile("vsrli.b $vr4, $vr0, 4"); + asm volatile("vsrli.b $vr5, $vr1, 4"); + asm volatile("vsrli.b $vr6, $vr2, 4"); + asm volatile("vsrli.b $vr7, $vr3, 4"); + /* vr12 - vr15: lower 4 bits of each byte of (P + Pxy) */ + asm volatile("vandi.b $vr12, $vr0, 0x0f"); + asm volatile("vandi.b $vr13, $vr1, 0x0f"); + asm volatile("vandi.b $vr14, $vr2, 0x0f"); + asm volatile("vandi.b $vr15, $vr3, 0x0f"); + /* lookup from pbmul[0] */ + asm volatile("vshuf.b $vr12, $vr22, $vr22, $vr12"); + asm volatile("vshuf.b $vr13, $vr22, $vr22, $vr13"); + asm volatile("vshuf.b $vr14, $vr22, $vr22, $vr14"); + asm volatile("vshuf.b $vr15, $vr22, $vr22, $vr15"); + /* lookup from pbmul[16] */ + asm volatile("vshuf.b $vr4, $vr23, $vr23, $vr4"); + asm volatile("vshuf.b $vr5, $vr23, $vr23, $vr5"); + asm volatile("vshuf.b $vr6, $vr23, $vr23, $vr6"); + asm volatile("vshuf.b $vr7, $vr23, $vr23, $vr7"); + /* vr4 - vr7: A(P + Pxy) */ + asm volatile("vxor.v $vr4, $vr4, $vr12"); + asm volatile("vxor.v $vr5, $vr5, $vr13"); + asm volatile("vxor.v $vr6, $vr6, $vr14"); + asm volatile("vxor.v $vr7, $vr7, $vr15"); + + /* vr4 - vr7: A(P + Pxy) + B(Q + Qxy) = Dx */ + asm volatile("vxor.v $vr4, $vr4, $vr16"); + asm volatile("vxor.v $vr5, $vr5, $vr17"); + asm volatile("vxor.v $vr6, $vr6, $vr18"); + asm volatile("vxor.v $vr7, $vr7, $vr19"); + asm volatile("vst $vr4, %0" : "=m" (dq[0])); + asm volatile("vst $vr5, %0" : "=m" (dq[16])); + asm volatile("vst $vr6, %0" : "=m" (dq[32])); + asm volatile("vst $vr7, %0" : "=m" (dq[48])); + + /* vr0 - vr3: P + Pxy + Dx = Dy */ + asm volatile("vxor.v $vr0, $vr0, $vr4"); + asm volatile("vxor.v $vr1, $vr1, $vr5"); + asm volatile("vxor.v $vr2, $vr2, $vr6"); + asm volatile("vxor.v $vr3, $vr3, $vr7"); + asm volatile("vst $vr0, %0" : "=m" (dp[0])); + asm volatile("vst $vr1, %0" : "=m" (dp[16])); + asm volatile("vst $vr2, %0" : "=m" (dp[32])); + asm volatile("vst $vr3, %0" : "=m" (dp[48])); + + bytes -= 64; + p += 64; + q += 64; + dp += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, + void **ptrs) +{ + u8 *p, *q, *dq; + const u8 *qmul; /* Q multiplier table */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dq; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; + + kernel_fpu_begin(); + + /* vr22, vr23: qmul */ + asm volatile("vld $vr22, %0" : : "m" (qmul[0])); + asm volatile("vld $vr23, %0" : : "m" (qmul[16])); + + while (bytes) { + /* vr0 - vr3: P + Dx */ + asm volatile("vld $vr0, %0" : : "m" (p[0])); + asm volatile("vld $vr1, %0" : : "m" (p[16])); + asm volatile("vld $vr2, %0" : : "m" (p[32])); + asm volatile("vld $vr3, %0" : : "m" (p[48])); + /* vr4 - vr7: Qx */ + asm volatile("vld $vr4, %0" : : "m" (dq[0])); + asm volatile("vld $vr5, %0" : : "m" (dq[16])); + asm volatile("vld $vr6, %0" : : "m" (dq[32])); + asm volatile("vld $vr7, %0" : : "m" (dq[48])); + /* vr4 - vr7: Q + Qx */ + asm volatile("vld $vr8, %0" : : "m" (q[0])); + asm volatile("vld $vr9, %0" : : "m" (q[16])); + asm volatile("vld $vr10, %0" : : "m" (q[32])); + asm volatile("vld $vr11, %0" : : "m" (q[48])); + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + + /* vr8 - vr11: higher 4 bits of each byte of (Q + Qx) */ + asm volatile("vsrli.b $vr8, $vr4, 4"); + asm volatile("vsrli.b $vr9, $vr5, 4"); + asm volatile("vsrli.b $vr10, $vr6, 4"); + asm volatile("vsrli.b $vr11, $vr7, 4"); + /* vr4 - vr7: lower 4 bits of each byte of (Q + Qx) */ + asm volatile("vandi.b $vr4, $vr4, 0x0f"); + asm volatile("vandi.b $vr5, $vr5, 0x0f"); + asm volatile("vandi.b $vr6, $vr6, 0x0f"); + asm volatile("vandi.b $vr7, $vr7, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("vshuf.b $vr4, $vr22, $vr22, $vr4"); + asm volatile("vshuf.b $vr5, $vr22, $vr22, $vr5"); + asm volatile("vshuf.b $vr6, $vr22, $vr22, $vr6"); + asm volatile("vshuf.b $vr7, $vr22, $vr22, $vr7"); + /* lookup from qmul[16] */ + asm volatile("vshuf.b $vr8, $vr23, $vr23, $vr8"); + asm volatile("vshuf.b $vr9, $vr23, $vr23, $vr9"); + asm volatile("vshuf.b $vr10, $vr23, $vr23, $vr10"); + asm volatile("vshuf.b $vr11, $vr23, $vr23, $vr11"); + /* vr4 - vr7: qmul(Q + Qx) = Dx */ + asm volatile("vxor.v $vr4, $vr4, $vr8"); + asm volatile("vxor.v $vr5, $vr5, $vr9"); + asm volatile("vxor.v $vr6, $vr6, $vr10"); + asm volatile("vxor.v $vr7, $vr7, $vr11"); + asm volatile("vst $vr4, %0" : "=m" (dq[0])); + asm volatile("vst $vr5, %0" : "=m" (dq[16])); + asm volatile("vst $vr6, %0" : "=m" (dq[32])); + asm volatile("vst $vr7, %0" : "=m" (dq[48])); + + /* vr0 - vr3: P + Dx + Dx = P */ + asm volatile("vxor.v $vr0, $vr0, $vr4"); + asm volatile("vxor.v $vr1, $vr1, $vr5"); + asm volatile("vxor.v $vr2, $vr2, $vr6"); + asm volatile("vxor.v $vr3, $vr3, $vr7"); + asm volatile("vst $vr0, %0" : "=m" (p[0])); + asm volatile("vst $vr1, %0" : "=m" (p[16])); + asm volatile("vst $vr2, %0" : "=m" (p[32])); + asm volatile("vst $vr3, %0" : "=m" (p[48])); + + bytes -= 64; + p += 64; + q += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +const struct raid6_recov_calls raid6_recov_lsx = { + .data2 = raid6_2data_recov_lsx, + .datap = raid6_datap_recov_lsx, + .valid = raid6_has_lsx, + .name = "lsx", + .priority = 1, +}; +#endif /* CONFIG_CPU_HAS_LSX */ + +#ifdef CONFIG_CPU_HAS_LASX +static int raid6_has_lasx(void) +{ + return cpu_has_lasx; +} + +static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, + int failb, void **ptrs) +{ + u8 *p, *q, *dp, *dq; + const u8 *pbmul; /* P multiplier table for B data */ + const u8 *qmul; /* Q multiplier table (for both) */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 2] = dp; + dq = (u8 *)ptrs[failb]; + ptrs[failb] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dp; + ptrs[failb] = dq; + ptrs[disks - 2] = p; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]]; + + kernel_fpu_begin(); + + /* + * xr20, xr21: qmul + * xr22, xr23: pbmul + */ + asm volatile("vld $vr20, %0" : : "m" (qmul[0])); + asm volatile("vld $vr21, %0" : : "m" (qmul[16])); + asm volatile("vld $vr22, %0" : : "m" (pbmul[0])); + asm volatile("vld $vr23, %0" : : "m" (pbmul[16])); + asm volatile("xvreplve0.q $xr20, $xr20"); + asm volatile("xvreplve0.q $xr21, $xr21"); + asm volatile("xvreplve0.q $xr22, $xr22"); + asm volatile("xvreplve0.q $xr23, $xr23"); + + while (bytes) { + /* xr0, xr1: Q */ + asm volatile("xvld $xr0, %0" : : "m" (q[0])); + asm volatile("xvld $xr1, %0" : : "m" (q[32])); + /* xr0, xr1: Q + Qxy */ + asm volatile("xvld $xr4, %0" : : "m" (dq[0])); + asm volatile("xvld $xr5, %0" : : "m" (dq[32])); + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + /* xr2, xr3: P */ + asm volatile("xvld $xr2, %0" : : "m" (p[0])); + asm volatile("xvld $xr3, %0" : : "m" (p[32])); + /* xr2, xr3: P + Pxy */ + asm volatile("xvld $xr4, %0" : : "m" (dp[0])); + asm volatile("xvld $xr5, %0" : : "m" (dp[32])); + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr4, xr5: higher 4 bits of each byte of (Q + Qxy) */ + asm volatile("xvsrli.b $xr4, $xr0, 4"); + asm volatile("xvsrli.b $xr5, $xr1, 4"); + /* xr0, xr1: lower 4 bits of each byte of (Q + Qxy) */ + asm volatile("xvandi.b $xr0, $xr0, 0x0f"); + asm volatile("xvandi.b $xr1, $xr1, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("xvshuf.b $xr0, $xr20, $xr20, $xr0"); + asm volatile("xvshuf.b $xr1, $xr20, $xr20, $xr1"); + /* lookup from qmul[16] */ + asm volatile("xvshuf.b $xr4, $xr21, $xr21, $xr4"); + asm volatile("xvshuf.b $xr5, $xr21, $xr21, $xr5"); + /* xr6, xr7: B(Q + Qxy) */ + asm volatile("xvxor.v $xr6, $xr4, $xr0"); + asm volatile("xvxor.v $xr7, $xr5, $xr1"); + + /* xr4, xr5: higher 4 bits of each byte of (P + Pxy) */ + asm volatile("xvsrli.b $xr4, $xr2, 4"); + asm volatile("xvsrli.b $xr5, $xr3, 4"); + /* xr0, xr1: lower 4 bits of each byte of (P + Pxy) */ + asm volatile("xvandi.b $xr0, $xr2, 0x0f"); + asm volatile("xvandi.b $xr1, $xr3, 0x0f"); + /* lookup from pbmul[0] */ + asm volatile("xvshuf.b $xr0, $xr22, $xr22, $xr0"); + asm volatile("xvshuf.b $xr1, $xr22, $xr22, $xr1"); + /* lookup from pbmul[16] */ + asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4"); + asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5"); + /* xr0, xr1: A(P + Pxy) */ + asm volatile("xvxor.v $xr0, $xr0, $xr4"); + asm volatile("xvxor.v $xr1, $xr1, $xr5"); + + /* xr0, xr1: A(P + Pxy) + B(Q + Qxy) = Dx */ + asm volatile("xvxor.v $xr0, $xr0, $xr6"); + asm volatile("xvxor.v $xr1, $xr1, $xr7"); + + /* xr2, xr3: P + Pxy + Dx = Dy */ + asm volatile("xvxor.v $xr2, $xr2, $xr0"); + asm volatile("xvxor.v $xr3, $xr3, $xr1"); + + asm volatile("xvst $xr0, %0" : "=m" (dq[0])); + asm volatile("xvst $xr1, %0" : "=m" (dq[32])); + asm volatile("xvst $xr2, %0" : "=m" (dp[0])); + asm volatile("xvst $xr3, %0" : "=m" (dp[32])); + + bytes -= 64; + p += 64; + q += 64; + dp += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, + void **ptrs) +{ + u8 *p, *q, *dq; + const u8 *qmul; /* Q multiplier table */ + + p = (u8 *)ptrs[disks - 2]; + q = (u8 *)ptrs[disks - 1]; + + /* + * Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = (u8 *)ptrs[faila]; + ptrs[faila] = (void *)raid6_empty_zero_page; + ptrs[disks - 1] = dq; + + raid6_call.gen_syndrome(disks, bytes, ptrs); + + /* Restore pointer table */ + ptrs[faila] = dq; + ptrs[disks - 1] = q; + + /* Now, pick the proper data tables */ + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; + + kernel_fpu_begin(); + + /* xr22, xr23: qmul */ + asm volatile("vld $vr22, %0" : : "m" (qmul[0])); + asm volatile("xvreplve0.q $xr22, $xr22"); + asm volatile("vld $vr23, %0" : : "m" (qmul[16])); + asm volatile("xvreplve0.q $xr23, $xr23"); + + while (bytes) { + /* xr0, xr1: P + Dx */ + asm volatile("xvld $xr0, %0" : : "m" (p[0])); + asm volatile("xvld $xr1, %0" : : "m" (p[32])); + /* xr2, xr3: Qx */ + asm volatile("xvld $xr2, %0" : : "m" (dq[0])); + asm volatile("xvld $xr3, %0" : : "m" (dq[32])); + /* xr2, xr3: Q + Qx */ + asm volatile("xvld $xr4, %0" : : "m" (q[0])); + asm volatile("xvld $xr5, %0" : : "m" (q[32])); + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr4, xr5: higher 4 bits of each byte of (Q + Qx) */ + asm volatile("xvsrli.b $xr4, $xr2, 4"); + asm volatile("xvsrli.b $xr5, $xr3, 4"); + /* xr2, xr3: lower 4 bits of each byte of (Q + Qx) */ + asm volatile("xvandi.b $xr2, $xr2, 0x0f"); + asm volatile("xvandi.b $xr3, $xr3, 0x0f"); + /* lookup from qmul[0] */ + asm volatile("xvshuf.b $xr2, $xr22, $xr22, $xr2"); + asm volatile("xvshuf.b $xr3, $xr22, $xr22, $xr3"); + /* lookup from qmul[16] */ + asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4"); + asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5"); + /* xr2, xr3: qmul(Q + Qx) = Dx */ + asm volatile("xvxor.v $xr2, $xr2, $xr4"); + asm volatile("xvxor.v $xr3, $xr3, $xr5"); + + /* xr0, xr1: P + Dx + Dx = P */ + asm volatile("xvxor.v $xr0, $xr0, $xr2"); + asm volatile("xvxor.v $xr1, $xr1, $xr3"); + + asm volatile("xvst $xr2, %0" : "=m" (dq[0])); + asm volatile("xvst $xr3, %0" : "=m" (dq[32])); + asm volatile("xvst $xr0, %0" : "=m" (p[0])); + asm volatile("xvst $xr1, %0" : "=m" (p[32])); + + bytes -= 64; + p += 64; + q += 64; + dq += 64; + } + + kernel_fpu_end(); +} + +const struct raid6_recov_calls raid6_recov_lasx = { + .data2 = raid6_2data_recov_lasx, + .datap = raid6_datap_recov_lasx, + .valid = raid6_has_lasx, + .name = "lasx", + .priority = 2, +}; +#endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index 4fb7700a741bdf289eda14ff0d2a6d27c9b8ea10..0a4e6e867f6038cf8c0a2afcbeca0cbfd3cb1779 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -34,6 +34,22 @@ ifeq ($(ARCH),aarch64) HAS_NEON = yes endif +ifeq ($(findstring ppc,$(ARCH)),ppc) + CFLAGS += -I../../../arch/powerpc/include + HAS_ALTIVEC := $(shell printf '$(pound)include \nvector int a;\n' |\ + gcc -c -x c - >/dev/null && rm ./-.o && echo yes) +endif + +ifeq ($(ARCH),loongarch64) + CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 + CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ + gcc -c -x assembler - >/dev/null 2>&1 && \ + rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) + CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ + gcc -c -x assembler - >/dev/null 2>&1 && \ + rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) +endif + ifeq ($(IS_X86),yes) OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o CFLAGS += -DCONFIG_X86 @@ -43,15 +59,12 @@ ifeq ($(IS_X86),yes) else ifeq ($(HAS_NEON),yes) OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else - HAS_ALTIVEC := $(shell printf '$(pound)include \nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) - ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -I../../../arch/powerpc/include - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o - endif +else ifeq ($(HAS_ALTIVEC),yes) + CFLAGS += -DCONFIG_ALTIVEC + OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ + vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o +else ifeq ($(ARCH),loongarch64) + OBJS += loongarch_simd.o recov_loongarch_simd.o endif .c.o: diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 2efc48444e770648ef4a4b0f25b6681c8148330c..2c57c9bb00c0d72b259f957bc4a47641bfffe6de 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -175,10 +175,16 @@ static __always_inline bool check_memory_region_inline(unsigned long addr, if (unlikely(addr + size < addr)) return !kasan_report(addr, size, write, ret_ip); +#ifndef __HAVE_ARCH_SHADOW_MAP if (unlikely((void *)addr < kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { return !kasan_report(addr, size, write, ret_ip); } +#else + if (unlikely(kasan_mem_to_shadow((void *)addr) == NULL)) { + return !kasan_report(addr, size, write, ret_ip); + } +#endif if (likely(!memory_is_poisoned(addr, size))) return true; diff --git a/mm/kasan/init.c b/mm/kasan/init.c index b8c6ec172bb22a5e4bda335d2a400b1b228886bc..dcfce7431a42fb1beabd47c35221cc4413dac76a 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -170,8 +170,11 @@ static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, if (!p) return -ENOMEM; } else { - pud_populate(&init_mm, pud, - early_alloc(PAGE_SIZE, NUMA_NO_NODE)); + p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); +#ifdef CONFIG_LOONGARCH + pmd_init((unsigned long)p, (unsigned long)invalid_pte_table); +#endif + pud_populate(&init_mm, pud, p); } } zero_pmd_populate(pud, addr, next); @@ -211,8 +214,11 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, if (!p) return -ENOMEM; } else { - p4d_populate(&init_mm, p4d, - early_alloc(PAGE_SIZE, NUMA_NO_NODE)); + p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); +#ifdef CONFIG_LOONGARCH + pud_init((unsigned long)p, (unsigned long)invalid_pmd_table); +#endif + p4d_populate(&init_mm, p4d, p); } } zero_pud_populate(p4d, addr, next); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index ac499456740f1f8b9805a991e8e71faa54256e8e..43befe1173867cce2d09e6b6acc8233790787b91 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -139,16 +139,20 @@ struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, struct kasan_free_meta *get_free_info(struct kmem_cache *cache, const void *object); +#ifndef kasan_shadow_to_mem static inline const void *kasan_shadow_to_mem(const void *shadow_addr) { return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) << KASAN_SHADOW_SCALE_SHIFT); } +#endif -static inline bool addr_has_shadow(const void *addr) +#ifndef addr_has_metadata +static inline bool addr_has_metadata(const void *addr) { return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START)); } +#endif void kasan_poison_shadow(const void *address, size_t size, u8 value); diff --git a/mm/sparse.c b/mm/sparse.c index 33406ea2ecc4480136e66d5ca7e85a8c1f89ca74..db0a7c53775b8fa9d0822e1f47391746beb8f269 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -809,6 +809,13 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, if (empty) { unsigned long section_nr = pfn_to_section_nr(pfn); + /* + * Mark the section invalid so that valid_section() + * return false. This prevents code from dereferencing + * ms->usage array. + */ + ms->section_mem_map &= ~SECTION_HAS_MEM_MAP; + /* * When removing an early section, the usage map is kept (as the * usage maps of other sections fall into the same page). It @@ -817,16 +824,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, * was allocated during boot. */ if (!PageReserved(virt_to_page(ms->usage))) { - kfree(ms->usage); - ms->usage = NULL; + kfree_rcu(ms->usage, rcu); + WRITE_ONCE(ms->usage, NULL); } memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); - /* - * Mark the section invalid so that valid_section() - * return false. This prevents code from dereferencing - * ms->usage array. - */ - ms->section_mem_map &= ~SECTION_HAS_MEM_MAP; } /* diff --git a/mm/util.c b/mm/util.c index 25bfda774f6fd5052109aa9d2d045d724747a5fe..7fd3c2bb3e4f539f37bec8331618bed8d1edcf09 100644 --- a/mm/util.c +++ b/mm/util.c @@ -686,6 +686,56 @@ static inline void *__page_rmapping(struct page *page) return (void *)mapping; } +/** + * __vmalloc_array - allocate memory for a virtually contiguous array. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate (see kmalloc). + */ +void *__vmalloc_array(size_t n, size_t size, gfp_t flags) +{ + size_t bytes; + + if (unlikely(check_mul_overflow(n, size, &bytes))) + return NULL; + return __vmalloc(bytes, flags); +} +EXPORT_SYMBOL(__vmalloc_array); + +/** + * vmalloc_array - allocate memory for a virtually contiguous array. + * @n: number of elements. + * @size: element size. + */ +void *vmalloc_array(size_t n, size_t size) +{ + return __vmalloc_array(n, size, GFP_KERNEL); +} +EXPORT_SYMBOL(vmalloc_array); + +/** + * __vcalloc - allocate and zero memory for a virtually contiguous array. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate (see kmalloc). + */ +void *__vcalloc(size_t n, size_t size, gfp_t flags) +{ + return __vmalloc_array(n, size, flags | __GFP_ZERO); +} +EXPORT_SYMBOL(__vcalloc); + +/** + * vcalloc - allocate and zero memory for a virtually contiguous array. + * @n: number of elements. + * @size: element size. + */ +void *vcalloc(size_t n, size_t size) +{ + return __vmalloc_array(n, size, GFP_KERNEL | __GFP_ZERO); +} +EXPORT_SYMBOL(vcalloc); + /* Neutral page->mapping pointer to address_space or anon_vma or other */ void *page_rmapping(struct page *page) { diff --git a/mm/zswapd.c b/mm/zswapd.c index 89383ab58c6ad3c5f8ab6ed29cf05f4824b0c36e..ca6fbc028e699d53ce96f6c4e6d4fe91a1bc2c7e 100644 --- a/mm/zswapd.c +++ b/mm/zswapd.c @@ -195,8 +195,8 @@ void zswapd_status_show(struct seq_file *m) { unsigned int buffers = calc_sys_cur_avail_buffers(); - seq_printf(m, "buffer_size:%u\n", buffers); - seq_printf(m, "recent_refault:%llu\n", anon_refault_ratio); + seq_printf(m, "buffer_size %u\n", buffers); + seq_printf(m, "recent_refault %llu\n", anon_refault_ratio); } pid_t get_zswapd_pid(void) diff --git a/net/8021q/vlan_netlink.c b/net/8021q/vlan_netlink.c index 0db85aeb119b88fae1abaf7d0a10019ba4044915..99b27777525792aa21e0a3bb83e7a3e1bcf78161 100644 --- a/net/8021q/vlan_netlink.c +++ b/net/8021q/vlan_netlink.c @@ -118,12 +118,16 @@ static int vlan_changelink(struct net_device *dev, struct nlattr *tb[], } if (data[IFLA_VLAN_INGRESS_QOS]) { nla_for_each_nested(attr, data[IFLA_VLAN_INGRESS_QOS], rem) { + if (nla_type(attr) != IFLA_VLAN_QOS_MAPPING) + continue; m = nla_data(attr); vlan_dev_set_ingress_priority(dev, m->to, m->from); } } if (data[IFLA_VLAN_EGRESS_QOS]) { nla_for_each_nested(attr, data[IFLA_VLAN_EGRESS_QOS], rem) { + if (nla_type(attr) != IFLA_VLAN_QOS_MAPPING) + continue; m = nla_data(attr); err = vlan_dev_set_egress_priority(dev, m->from, m->to); if (err) diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index b9cf5bc9364c1370c625c9fc20fa4e9529c08222..c8c1cd55c0eb097136563add3fcd9a5949781384 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -3839,7 +3839,11 @@ void hci_unregister_dev(struct hci_dev *hdev) list_del(&hdev->list); write_unlock(&hci_dev_list_lock); + cancel_work_sync(&hdev->rx_work); + cancel_work_sync(&hdev->cmd_work); + cancel_work_sync(&hdev->tx_work); cancel_work_sync(&hdev->power_on); + cancel_work_sync(&hdev->error_reset); if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { hci_suspend_clear_tasks(hdev); diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index ef5c174102d5e3114e54d31c8c76d43c42d77978..82d3f1204ee48e8da36d1554a74fcda40e233438 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -1014,13 +1014,19 @@ static void delayed_work(struct work_struct *work) struct ceph_mon_client *monc = container_of(work, struct ceph_mon_client, delayed_work.work); - dout("monc delayed_work\n"); mutex_lock(&monc->mutex); + dout("%s mon%d\n", __func__, monc->cur_mon); + if (monc->cur_mon < 0) { + goto out; + } + if (monc->hunting) { dout("%s continuing hunt\n", __func__); reopen_session(monc); } else { int is_auth = ceph_auth_is_authenticated(monc->auth); + + dout("%s is_authed %d\n", __func__, is_auth); if (ceph_con_keepalive_expired(&monc->con, CEPH_MONC_PING_TIMEOUT)) { dout("monc keepalive timeout\n"); @@ -1045,6 +1051,8 @@ static void delayed_work(struct work_struct *work) } } __schedule_delayed(monc); + +out: mutex_unlock(&monc->mutex); } @@ -1157,13 +1165,15 @@ EXPORT_SYMBOL(ceph_monc_init); void ceph_monc_stop(struct ceph_mon_client *monc) { dout("stop\n"); - cancel_delayed_work_sync(&monc->delayed_work); mutex_lock(&monc->mutex); __close_session(monc); + monc->hunting = false; monc->cur_mon = -1; mutex_unlock(&monc->mutex); + cancel_delayed_work_sync(&monc->delayed_work); + /* * flush msgr queue before we destroy ourselves to ensure that: * - any work that references our embedded con is finished. diff --git a/net/core/lowpower_protocol.c b/net/core/lowpower_protocol.c index 96f0e8e92b202cf2f6bf57be7bf016f57f01ace9..60d564550114ce51624414f93cba75fad08b7920 100644 --- a/net/core/lowpower_protocol.c +++ b/net/core/lowpower_protocol.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -17,6 +19,17 @@ #include static atomic_t g_foreground_uid = ATOMIC_INIT(FOREGROUND_UID_INIT); +#define OPT_LEN 3 +#define TO_DECIMAL 10 +#define LIST_MAX 500 +#define DECIMAL_CHAR_NUM 10 // u32 decimal characters (4,294,967,295) +static DEFINE_RWLOCK(g_dpa_rwlock); +static u32 g_dpa_uid_list_cnt; +static struct list_head g_dpa_uid_list; +struct dpa_node { + struct list_head list_node; + uid_t uid; +}; static void foreground_uid_atomic_set(uid_t val) { @@ -28,6 +41,7 @@ static uid_t foreground_uid_atomic_read(void) return (uid_t)atomic_read(&g_foreground_uid); } +// cat /proc/net/foreground_uid static int foreground_uid_show(struct seq_file *seq, void *v) { uid_t uid = foreground_uid_atomic_read(); @@ -40,7 +54,7 @@ static int foreground_uid_show(struct seq_file *seq, void *v) static int foreground_uid_write(struct file *file, char *buf, size_t size) { char *p = buf; - uid_t uid = simple_strtoul(p, &p, 10); + uid_t uid = simple_strtoul(p, &p, TO_DECIMAL); if (!p) return -EINVAL; @@ -49,6 +63,231 @@ static int foreground_uid_write(struct file *file, char *buf, size_t size) return 0; } +// cat /proc/net/dpa_uid +static int dpa_uid_show(struct seq_file *seq, void *v) +{ + struct dpa_node *node = NULL; + struct dpa_node *tmp_node = NULL; + + read_lock(&g_dpa_rwlock); + seq_printf(seq, "uid list num: %u\n", g_dpa_uid_list_cnt); + list_for_each_entry_safe(node, tmp_node, &g_dpa_uid_list, list_node) + seq_printf(seq, "%u\n", node->uid); + read_unlock(&g_dpa_rwlock); + return 0; +} + +// echo "add xx yy zz" > /proc/net/dpa_uid +// echo "del xx yy zz" > /proc/net/dpa_uid +static int dpa_uid_add(uid_t uid); +static int dpa_uid_del(uid_t uid); +static int get_dpa_uids(char *buf, size_t size, u32 *uid_list, + u32 index_max, u32 *index); +static int dpa_uid_write(struct file *file, char *buf, size_t size) +{ + u32 dpa_list[LIST_MAX]; + u32 index = 0; + int ret = -EINVAL; + int i; + + if (get_dpa_uids(buf, size, dpa_list, LIST_MAX, &index) != 0) { + pr_err("[dpa-uid-cfg] fail to parse dpa uids\n"); + return ret; + } + + if (strncmp(buf, "add", OPT_LEN) == 0) { + for (i = 0; i < index; i++) { + ret = dpa_uid_add(dpa_list[i]); + if (ret != 0) { + pr_err("[dpa-uid-cfg] add fail, index=%u\n", i); + return ret; + } + } + } else if (strncmp(buf, "del", OPT_LEN) == 0) { + for (i = 0; i < index; i++) { + ret = dpa_uid_del(dpa_list[i]); + if (ret != 0) { + pr_err("[dpa-uid-cfg] del fail, index=%u\n", i); + return ret; + } + } + } else { + pr_err("[dpa-uid-cfg] cmd unknown\n"); + } + return ret; +} + +static int dpa_uid_add(uid_t uid) +{ + bool exist = false; + struct dpa_node *node = NULL; + struct dpa_node *tmp_node = NULL; + + write_lock(&g_dpa_rwlock); + if (g_dpa_uid_list_cnt >= LIST_MAX) { + write_unlock(&g_dpa_rwlock); + return -EFBIG; + } + + list_for_each_entry_safe(node, tmp_node, &g_dpa_uid_list, list_node) { + if (node->uid == uid) { + exist = true; + break; + } + } + + if (!exist) { + node = kzalloc(sizeof(*node), GFP_ATOMIC); + if (node) { + node->uid = uid; + list_add_tail(&node->list_node, &g_dpa_uid_list); + g_dpa_uid_list_cnt++; + } + } + write_unlock(&g_dpa_rwlock); + return 0; +} + +static int dpa_uid_del(uid_t uid) +{ + struct dpa_node *node = NULL; + struct dpa_node *tmp_node = NULL; + + write_lock(&g_dpa_rwlock); + list_for_each_entry_safe(node, tmp_node, &g_dpa_uid_list, list_node) { + if (node->uid == uid) { + list_del(&node->list_node); + if (g_dpa_uid_list_cnt) + --g_dpa_uid_list_cnt; + break; + } + } + write_unlock(&g_dpa_rwlock); + return 0; +} + +static uid_t parse_single_uid(char *begin, char *end) +{ + char *cur = NULL; + uid_t uid = 0; + u32 len = end - begin; + + // u32 decimal characters (4,294,967,295) + if (len > DECIMAL_CHAR_NUM) { + pr_err("[dpa-uid-cfg] single uid len(%u) overflow\n", len); + return uid; + } + + cur = begin; + while (cur < end) { + if (*cur < '0' || *cur > '9') { + pr_err("[dpa-uid-cfg] invalid character '%c'\n", *cur); + return uid; + } + cur++; + } + + uid = simple_strtoul(begin, &begin, TO_DECIMAL); + if (!begin || !uid) { + pr_err("[dpa-uid-cfg] fail to change str to data"); + return uid; + } + + return uid; +} + +static int parse_uids(char *args, u32 args_len, u32 *uid_list, + u32 index_max, u32 *index) +{ + char *begin = args; + char *end = strchr(args, ' '); + uid_t uid = 0; + u32 len = 0; + + while (end) { + // cur decimal characters cnt + ' ' or '\n' + len += end - begin + 1; + if (len > args_len || *index > index_max) { + pr_err("[dpa-uid-cfg] str len(%u) or index(%u) overflow\n", + len, *index); + return -EINVAL; + } + + uid = parse_single_uid(begin, end); + if (!uid) + return -EINVAL; + uid_list[(*index)++] = uid; + begin = ++end; // next decimal characters (skip ' ' or '\n') + end = strchr(begin, ' '); + } + + // find last uid characters + end = strchr(begin, '\n'); + if (!end) { + pr_err("[dpa-uid-cfg] last character is not '\\n'"); + return -EINVAL; + } + + // cur decimal characters cnt + ' ' or '\n' + len += end - begin + 1; + if (len > args_len || *index > index_max) { + pr_err("[dpa-uid-cfg] str len(%u) or last index(%u) overflow\n", + len, *index); + return -EINVAL; + } + uid = parse_single_uid(begin, end); + if (!uid) + return -EINVAL; + uid_list[(*index)++] = uid; + return 0; +} + +static int get_dpa_uids(char *buf, size_t size, u32 *uid_list, + u32 index_max, u32 *index) +{ + char *args = NULL; + u32 opt_len; + u32 data_len; + + // split into cmd and argslist + args = strchr(buf, ' '); + if (!args) { + pr_err("[dpa-uid-cfg] cmd fmt invalid\n"); + return -EINVAL; + } + + // cmd is add or del, len is 3 + opt_len = args - buf; + if (opt_len != OPT_LEN) { + pr_err("[dpa-uid-cfg] cmd len invalid\n"); + return -EINVAL; + } + + data_len = size - (opt_len + 1); + return parse_uids(args + 1, data_len, uid_list, index_max, index); +} + +bool dpa_uid_match(uid_t kuid) +{ + bool match = false; + struct dpa_node *node = NULL; + struct dpa_node *tmp_node = NULL; + + if (kuid == 0) + return match; + + read_lock(&g_dpa_rwlock); + list_for_each_entry_safe(node, tmp_node, &g_dpa_uid_list, list_node) { + if (node->uid == kuid) { + match = true; + break; + } + } + read_unlock(&g_dpa_rwlock); + return match; +} +EXPORT_SYMBOL(dpa_uid_match); + // call this fun in net/ipv4/af_inet.c inet_init_net() void __net_init lowpower_protocol_net_init(struct net *net) { @@ -57,7 +296,15 @@ void __net_init lowpower_protocol_net_init(struct net *net) foreground_uid_show, foreground_uid_write, NULL)) - pr_err("fail to register /proc/net/foreground_uid"); + pr_err("fail to create /proc/net/foreground_uid"); + + INIT_LIST_HEAD(&g_dpa_uid_list); + if (!proc_create_net_single_write("dpa_uid", 0644, + net->proc_net, + dpa_uid_show, + dpa_uid_write, + NULL)) + pr_err("fail to create /proc/net/dpa_uid"); } static bool foreground_uid_match(struct net *net, struct sock *sk) diff --git a/net/core/xdp.c b/net/core/xdp.c index b8d7fa47d293c3151032e501877fbf76a5636a2d..fd98d6059007c1e7fbfdd1350044fa2a495fb596 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -110,26 +110,37 @@ static void mem_allocator_disconnect(void *allocator) mutex_unlock(&mem_id_lock); } -void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq) +void xdp_unreg_mem_model(struct xdp_mem_info *mem) { struct xdp_mem_allocator *xa; - int id = xdp_rxq->mem.id; + int type = mem->type; + int id = mem->id; - if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { - WARN(1, "Missing register, driver bug"); - return; - } + /* Reset mem info to defaults */ + mem->id = 0; + mem->type = 0; if (id == 0) return; - if (xdp_rxq->mem.type == MEM_TYPE_PAGE_POOL) { + if (type == MEM_TYPE_PAGE_POOL) { rcu_read_lock(); xa = rhashtable_lookup(mem_id_ht, &id, mem_id_rht_params); page_pool_destroy(xa->page_pool); rcu_read_unlock(); } } +EXPORT_SYMBOL_GPL(xdp_unreg_mem_model); + +void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq) +{ + if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { + WARN(1, "Missing register, driver bug"); + return; + } + + xdp_unreg_mem_model(&xdp_rxq->mem); +} EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg_mem_model); void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq) @@ -144,10 +155,6 @@ void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq) xdp_rxq->reg_state = REG_STATE_UNREGISTERED; xdp_rxq->dev = NULL; - - /* Reset mem info to defaults */ - xdp_rxq->mem.id = 0; - xdp_rxq->mem.type = 0; } EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg); @@ -259,28 +266,24 @@ static bool __is_supported_mem_type(enum xdp_mem_type type) return true; } -int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, - enum xdp_mem_type type, void *allocator) +static struct xdp_mem_allocator *__xdp_reg_mem_model(struct xdp_mem_info *mem, + enum xdp_mem_type type, + void *allocator) { struct xdp_mem_allocator *xdp_alloc; gfp_t gfp = GFP_KERNEL; int id, errno, ret; void *ptr; - if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { - WARN(1, "Missing register, driver bug"); - return -EFAULT; - } - if (!__is_supported_mem_type(type)) - return -EOPNOTSUPP; + return ERR_PTR(-EOPNOTSUPP); - xdp_rxq->mem.type = type; + mem->type = type; if (!allocator) { if (type == MEM_TYPE_PAGE_POOL) - return -EINVAL; /* Setup time check page_pool req */ - return 0; + return ERR_PTR(-EINVAL); /* Setup time check page_pool req */ + return NULL; } /* Delay init of rhashtable to save memory if feature isn't used */ @@ -288,15 +291,13 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, mutex_lock(&mem_id_lock); ret = __mem_id_init_hash_table(); mutex_unlock(&mem_id_lock); - if (ret < 0) { - WARN_ON(1); - return ret; - } + if (ret < 0) + return ERR_PTR(ret); } xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp); if (!xdp_alloc) - return -ENOMEM; + return ERR_PTR(-ENOMEM); mutex_lock(&mem_id_lock); id = __mem_id_cyclic_get(gfp); @@ -304,15 +305,15 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, errno = id; goto err; } - xdp_rxq->mem.id = id; - xdp_alloc->mem = xdp_rxq->mem; + mem->id = id; + xdp_alloc->mem = *mem; xdp_alloc->allocator = allocator; /* Insert allocator into ID lookup table */ ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node); if (IS_ERR(ptr)) { - ida_simple_remove(&mem_id_pool, xdp_rxq->mem.id); - xdp_rxq->mem.id = 0; + ida_simple_remove(&mem_id_pool, mem->id); + mem->id = 0; errno = PTR_ERR(ptr); goto err; } @@ -322,13 +323,44 @@ int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, mutex_unlock(&mem_id_lock); - trace_mem_connect(xdp_alloc, xdp_rxq); - return 0; + return xdp_alloc; err: mutex_unlock(&mem_id_lock); kfree(xdp_alloc); - return errno; + return ERR_PTR(errno); } + +int xdp_reg_mem_model(struct xdp_mem_info *mem, + enum xdp_mem_type type, void *allocator) +{ + struct xdp_mem_allocator *xdp_alloc; + + xdp_alloc = __xdp_reg_mem_model(mem, type, allocator); + if (IS_ERR(xdp_alloc)) + return PTR_ERR(xdp_alloc); + return 0; +} +EXPORT_SYMBOL_GPL(xdp_reg_mem_model); + +int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, + enum xdp_mem_type type, void *allocator) +{ + struct xdp_mem_allocator *xdp_alloc; + + if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { + WARN(1, "Missing register, driver bug"); + return -EFAULT; + } + + xdp_alloc = __xdp_reg_mem_model(&xdp_rxq->mem, type, allocator); + if (IS_ERR(xdp_alloc)) + return PTR_ERR(xdp_alloc); + + if (trace_mem_connect_enabled() && xdp_alloc) + trace_mem_connect(xdp_alloc, xdp_rxq); + return 0; +} + EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); /* XDP RX runs under NAPI protection, and in different delivery error diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c index 8324e9f970668d439a5be589a1a5b88b39b00b86..26a9d8434c234a1880b65960a8d4de000a361934 100644 --- a/net/dns_resolver/dns_key.c +++ b/net/dns_resolver/dns_key.c @@ -104,7 +104,7 @@ dns_resolver_preparse(struct key_preparsed_payload *prep) const struct dns_server_list_v1_header *v1; /* It may be a server list. */ - if (datalen <= sizeof(*v1)) + if (datalen < sizeof(*v1)) return -EINVAL; v1 = (const struct dns_server_list_v1_header *)data; diff --git a/net/ethtool/features.c b/net/ethtool/features.c index 1c9f4df273bd55287aa08e5da82c32f1bc96a1ed..faccab84d8656adbd7aaa7e2a6c287b8f043879a 100644 --- a/net/ethtool/features.c +++ b/net/ethtool/features.c @@ -235,17 +235,20 @@ int ethnl_set_features(struct sk_buff *skb, struct genl_info *info) dev = req_info.dev; rtnl_lock(); + ret = ethnl_ops_begin(dev); + if (ret < 0) + goto out_rtnl; ethnl_features_to_bitmap(old_active, dev->features); ethnl_features_to_bitmap(old_wanted, dev->wanted_features); ret = ethnl_parse_bitset(req_wanted, req_mask, NETDEV_FEATURE_COUNT, tb[ETHTOOL_A_FEATURES_WANTED], netdev_features_strings, info->extack); if (ret < 0) - goto out_rtnl; + goto out_ops; if (ethnl_bitmap_to_features(req_mask) & ~NETIF_F_ETHTOOL_BITS) { GENL_SET_ERR_MSG(info, "attempt to change non-ethtool features"); ret = -EINVAL; - goto out_rtnl; + goto out_ops; } /* set req_wanted bits not in req_mask from old_wanted */ @@ -282,6 +285,8 @@ int ethnl_set_features(struct sk_buff *skb, struct genl_info *info) if (mod) netdev_features_change(dev); +out_ops: + ethnl_ops_complete(dev); out_rtnl: rtnl_unlock(); dev_put(dev); diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 939792a3881461275b9c7fae5b3a5e0881a59584..be1976536f1c00e0b8e525207c79004628b6f8fc 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -261,7 +261,9 @@ static int __net_init ipmr_rules_init(struct net *net) return 0; err2: + rtnl_lock(); ipmr_free_table(mrt); + rtnl_unlock(); err1: fib_rules_unregister(ops); return err; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index deb84d431e5a71023b43c1ce9d7c9fe82d4c83fc..f767d19c5c7b6c2948fc0fce0dc84de9ac96f666 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -739,6 +739,7 @@ void tcp_push(struct sock *sk, int flags, int mss_now, if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING); set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); + smp_mb__after_atomic(); } /* It is possible TX completion already happened * before we set TSQ_THROTTLED. diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c index f823a15b973c416cba1307d4aa6495ce18c4e469..95b5ac082a2f4e43ac362fc06c3fb53d4bbea0f9 100644 --- a/net/ipv4/tcp_metrics.c +++ b/net/ipv4/tcp_metrics.c @@ -619,6 +619,7 @@ static const struct nla_policy tcp_metrics_nl_policy[TCP_METRICS_ATTR_MAX + 1] = [TCP_METRICS_ATTR_ADDR_IPV4] = { .type = NLA_U32, }, [TCP_METRICS_ATTR_ADDR_IPV6] = { .type = NLA_BINARY, .len = sizeof(struct in6_addr), }, + [TCP_METRICS_ATTR_SADDR_IPV4] = { .type = NLA_U32, }, /* Following attributes are not received for GET/DEL, * we keep them for reference */ diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 476f79f1563a8aa993ff635f09a7bf338f324a2d..cc93541770f59308acdeadc1d67eb0fa2cb0ac86 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -317,6 +317,8 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, goto fail_unlock; } + sock_set_flag(sk, SOCK_RCU_FREE); + sk_add_node_rcu(sk, &hslot->head); hslot->count++; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); @@ -333,7 +335,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, hslot2->count++; spin_unlock(&hslot2->lock); } - sock_set_flag(sk, SOCK_RCU_FREE); + error = 0; fail_unlock: spin_unlock_bh(&hslot->lock); @@ -602,6 +604,13 @@ static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, } DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key); +EXPORT_SYMBOL(udp_encap_needed_key); + +#if IS_ENABLED(CONFIG_IPV6) +DEFINE_STATIC_KEY_FALSE(udpv6_encap_needed_key); +EXPORT_SYMBOL(udpv6_encap_needed_key); +#endif + void udp_encap_enable(void) { static_branch_inc(&udp_encap_needed_key); diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index f4b8e56068e068855a4be6ace128ac0ae158246b..17e4e537365af79465217d41e2f82d520d9b4d6b 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -512,6 +512,11 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, unsigned int off = skb_gro_offset(skb); int flush = 1; + /* We can do L4 aggregation only if the packet can't land in a tunnel + * otherwise we could corrupt the inner stream. Detecting such packets + * cannot be foolproof and the aggregation might still happen in some + * cases. Such packets should be caught in udp_unexpected_gso later. + */ NAPI_GRO_CB(skb)->is_flist = 0; if (skb->dev->features & NETIF_F_GRO_FRAGLIST) NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled: 1; diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 0979a4b15f2620f4c15e35e3c3d2581cb8246328..1a294c45dc1f17928951ca3652a8370bbe8a821a 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -197,6 +197,9 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, if (INET_PROTOSW_REUSE & answer_flags) sk->sk_reuse = SK_CAN_REUSE; + if (INET_PROTOSW_ICSK & answer_flags) + inet_init_csk_locks(sk); + inet = inet_sk(sk); inet->is_icsk = (INET_PROTOSW_ICSK & answer_flags) != 0; diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 23881e5ebb3d4a60c96ecefd12355651c15677c3..7883bdb5171f1b1f23bcdf86ab13a9f2083e5459 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -960,6 +960,7 @@ static void __fib6_drop_pcpu_from(struct fib6_nh *fib6_nh, if (!fib6_nh->rt6i_pcpu) return; + rcu_read_lock(); /* release the reference to this fib entry from * all of its cached pcpu routes */ @@ -968,7 +969,9 @@ static void __fib6_drop_pcpu_from(struct fib6_nh *fib6_nh, struct rt6_info *pcpu_rt; ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); - pcpu_rt = *ppcpu_rt; + + /* Paired with xchg() in rt6_get_pcpu_route() */ + pcpu_rt = READ_ONCE(*ppcpu_rt); /* only dropping the 'from' reference if the cached route * is using 'match'. The cached pcpu_rt->from only changes @@ -982,6 +985,7 @@ static void __fib6_drop_pcpu_from(struct fib6_nh *fib6_nh, fib6_info_release(from); } } + rcu_read_unlock(); } struct fib6_nh_pcpu_arg { diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index a8bdaec4a02b4041350ec8d322d7026775efaec9..c758d0cc6146db46feb9d00f83372e75510e5926 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -248,7 +248,9 @@ static int __net_init ip6mr_rules_init(struct net *net) return 0; err2: + rtnl_lock(); ip6mr_free_table(mrt); + rtnl_unlock(); err1: fib_rules_unregister(ops); return err; diff --git a/net/ipv6/route.c b/net/ipv6/route.c index f41f9232857c102404f76e7dd03a9d8c93475533..7e3eb0af4b0e3cc4f82a6decef9df1fbcf96d37c 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -632,6 +632,8 @@ static void rt6_probe(struct fib6_nh *fib6_nh) rcu_read_lock_bh(); last_probe = READ_ONCE(fib6_nh->last_probe); idev = __in6_dev_get(dev); + if (!idev) + goto out; neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); if (neigh) { if (neigh->nud_state & NUD_VALID) @@ -1396,6 +1398,7 @@ static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) struct rt6_info *prev, **p; p = this_cpu_ptr(res->nh->rt6i_pcpu); + /* Paired with READ_ONCE() in __fib6_drop_pcpu_from() */ prev = xchg(p, NULL); if (prev) { dst_dev_put(&prev->dst); @@ -3487,7 +3490,7 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, if (!dev) goto out; - if (idev->cnf.disable_ipv6) { + if (!idev || idev->cnf.disable_ipv6) { NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); err = -EACCES; goto out; diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 5385037209a6bbb836556d6f0f277a69d9ac24ed..b5d879f2501da69d920e44c8340163bd6e18fa4c 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -474,7 +474,7 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, goto try_again; } -DEFINE_STATIC_KEY_FALSE(udpv6_encap_needed_key); +DECLARE_STATIC_KEY_FALSE(udpv6_encap_needed_key); void udpv6_encap_enable(void) { static_branch_inc(&udpv6_encap_needed_key); diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 4c3aa97f23faac3434ff44bccc6bd138d2ef839e..7c903e0e446cb7676da8b84dfe64bd2812e07d8f 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -57,12 +57,18 @@ static int xfrm6_get_saddr(struct net *net, int oif, { struct dst_entry *dst; struct net_device *dev; + struct inet6_dev *idev; dst = xfrm6_dst_lookup(net, 0, oif, NULL, daddr, mark); if (IS_ERR(dst)) return -EHOSTUNREACH; - dev = ip6_dst_idev(dst)->dev; + idev = ip6_dst_idev(dst); + if (!idev) { + dst_release(dst); + return -EHOSTUNREACH; + } + dev = idev->dev; ipv6_dev_get_saddr(dev_net(dev), dev, &daddr->in6, 0, &saddr->in6); dst_release(dst); return 0; diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 01e26698285a05c4e14d293d9fd8b40b41a688b8..f9785150bfcc5c54938ea2c63d75abd482dbf7e4 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -927,14 +927,15 @@ static int llc_ui_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, */ static int llc_ui_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { + DECLARE_SOCKADDR(struct sockaddr_llc *, addr, msg->msg_name); struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); - DECLARE_SOCKADDR(struct sockaddr_llc *, addr, msg->msg_name); int flags = msg->msg_flags; int noblock = flags & MSG_DONTWAIT; + int rc = -EINVAL, copied = 0, hdrlen, hh_len; struct sk_buff *skb = NULL; + struct net_device *dev; size_t size = 0; - int rc = -EINVAL, copied = 0, hdrlen; dprintk("%s: sending from %02X to %02X\n", __func__, llc->laddr.lsap, llc->daddr.lsap); @@ -954,22 +955,29 @@ static int llc_ui_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) if (rc) goto out; } - hdrlen = llc->dev->hard_header_len + llc_ui_header_len(sk, addr); + dev = llc->dev; + hh_len = LL_RESERVED_SPACE(dev); + hdrlen = llc_ui_header_len(sk, addr); size = hdrlen + len; - if (size > llc->dev->mtu) - size = llc->dev->mtu; + size = min_t(size_t, size, READ_ONCE(dev->mtu)); copied = size - hdrlen; rc = -EINVAL; if (copied < 0) goto out; release_sock(sk); - skb = sock_alloc_send_skb(sk, size, noblock, &rc); + skb = sock_alloc_send_skb(sk, hh_len + size, noblock, &rc); lock_sock(sk); if (!skb) goto out; - skb->dev = llc->dev; + if (sock_flag(sk, SOCK_ZAPPED) || + llc->dev != dev || + hdrlen != llc_ui_header_len(sk, addr) || + hh_len != LL_RESERVED_SPACE(dev) || + size > READ_ONCE(dev->mtu)) + goto out; + skb->dev = dev; skb->protocol = llc_proto_type(addr->sllc_arphrd); - skb_reserve(skb, hdrlen); + skb_reserve(skb, hh_len + hdrlen); rc = memcpy_from_msg(skb_put(skb, copied), msg, copied); if (rc) goto out; diff --git a/net/llc/llc_core.c b/net/llc/llc_core.c index 64d4bef04e7302db8bec14965ed92ee5867b5a1d..4900a27b51768d558e6c5898aad2a234896f3249 100644 --- a/net/llc/llc_core.c +++ b/net/llc/llc_core.c @@ -135,22 +135,15 @@ static struct packet_type llc_packet_type __read_mostly = { .func = llc_rcv, }; -static struct packet_type llc_tr_packet_type __read_mostly = { - .type = cpu_to_be16(ETH_P_TR_802_2), - .func = llc_rcv, -}; - static int __init llc_init(void) { dev_add_pack(&llc_packet_type); - dev_add_pack(&llc_tr_packet_type); return 0; } static void __exit llc_exit(void) { dev_remove_pack(&llc_packet_type); - dev_remove_pack(&llc_tr_packet_type); } module_init(llc_init); diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c index d936ef0c17a3752f64bcac3046a65634342794d2..72ecce377d174448a8d571c0afddf789ecd2f697 100644 --- a/net/mac80211/mesh_pathtbl.c +++ b/net/mac80211/mesh_pathtbl.c @@ -723,10 +723,23 @@ void mesh_path_discard_frame(struct ieee80211_sub_if_data *sdata, */ void mesh_path_flush_pending(struct mesh_path *mpath) { + struct ieee80211_sub_if_data *sdata = mpath->sdata; + struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh; + struct mesh_preq_queue *preq, *tmp; struct sk_buff *skb; while ((skb = skb_dequeue(&mpath->frame_queue)) != NULL) mesh_path_discard_frame(mpath->sdata, skb); + + spin_lock_bh(&ifmsh->mesh_preq_queue_lock); + list_for_each_entry_safe(preq, tmp, &ifmsh->preq_queue.list, list) { + if (ether_addr_equal(mpath->dst, preq->dst)) { + list_del(&preq->list); + kfree(preq); + --ifmsh->preq_queue_len; + } + } + spin_unlock_bh(&ifmsh->mesh_preq_queue_lock); } /** diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 44bd03c6b84734ac7d10d2d492afe96c9f658c78..f7637176d719dc1ee38d6e36ea13feeb25d32371 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -1343,7 +1343,7 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta) skb_queue_head_init(&pending); /* sync with ieee80211_tx_h_unicast_ps_buf */ - spin_lock(&sta->ps_lock); + spin_lock_bh(&sta->ps_lock); /* Send all buffered frames to the station */ for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { int count = skb_queue_len(&pending), tmp; @@ -1372,7 +1372,7 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta) */ clear_sta_flag(sta, WLAN_STA_PSPOLL); clear_sta_flag(sta, WLAN_STA_UAPSD); - spin_unlock(&sta->ps_lock); + spin_unlock_bh(&sta->ps_lock); atomic_dec(&ps->num_sta_ps); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index cd2130e98836b4e4fdbf0f65617634820f911505..c87dbc8970023473df5a78e2786062fa37dc60b1 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -271,7 +271,7 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, skb->dev = dst->dev; icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0); - __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); + IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS); return false; } @@ -286,7 +286,7 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, { if (ip_hdr(skb)->ttl <= 1) { /* Tell the sender its packet died... */ - __IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS); + IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS); icmp_send(skb, ICMP_TIME_EXCEEDED, ICMP_EXC_TTL, 0); return false; } diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index dada0a0e8bd70bc57942d9744dc21664fdc9ac13..f586e8b3c6cfac18c920750879493a6edfd593aa 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -25,6 +25,7 @@ #include #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-")) +#define NFT_SET_MAX_ANONLEN 16 unsigned int nf_tables_net_id __read_mostly; @@ -3930,6 +3931,9 @@ static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set, if (p[1] != 'd' || strchr(p + 2, '%')) return -EINVAL; + if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN) + return -EINVAL; + inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL); if (inuse == NULL) return -ENOMEM; @@ -4302,8 +4306,8 @@ static int nft_set_desc_concat_parse(const struct nlattr *attr, static int nft_set_desc_concat(struct nft_set_desc *desc, const struct nlattr *nla) { + u32 num_regs = 0, key_num_regs = 0; struct nlattr *attr; - u32 num_regs = 0; int rem, err, i; nla_for_each_nested(attr, nla, rem) { @@ -4318,6 +4322,10 @@ static int nft_set_desc_concat(struct nft_set_desc *desc, for (i = 0; i < desc->field_count; i++) num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32)); + key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32)); + if (key_num_regs != num_regs) + return -EINVAL; + if (num_regs > NFT_REG32_COUNT) return -E2BIG; @@ -4462,8 +4470,12 @@ static int nf_tables_newset(struct net *net, struct sock *nlsk, if (err < 0) return err; - if (desc.field_count > 1 && !(flags & NFT_SET_CONCAT)) + if (desc.field_count > 1) { + if (!(flags & NFT_SET_CONCAT)) + return -EINVAL; + } else if (flags & NFT_SET_CONCAT) { return -EINVAL; + } } else if (flags & NFT_SET_CONCAT) { return -EINVAL; } @@ -5007,7 +5019,7 @@ static int nf_tables_dump_setelem(const struct nft_ctx *ctx, const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv); struct nft_set_dump_args *args; - if (nft_set_elem_expired(ext)) + if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext)) return 0; args = container_of(iter, struct nft_set_dump_args, iter); diff --git a/net/netfilter/nft_chain_filter.c b/net/netfilter/nft_chain_filter.c index 7a9aa57b195bf798ecb46e3f07e1050846464ea6..a18582a4ecf347396e4b60d17e24b48811dead3c 100644 --- a/net/netfilter/nft_chain_filter.c +++ b/net/netfilter/nft_chain_filter.c @@ -358,9 +358,10 @@ static int nf_tables_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct nft_base_chain *basechain; struct nftables_pernet *nft_net; - struct nft_table *table; struct nft_chain *chain, *nr; + struct nft_table *table; struct nft_ctx ctx = { .net = dev_net(dev), }; @@ -372,7 +373,8 @@ static int nf_tables_netdev_event(struct notifier_block *this, nft_net = net_generic(ctx.net, nf_tables_net_id); mutex_lock(&nft_net->commit_mutex); list_for_each_entry(table, &nft_net->tables, list) { - if (table->family != NFPROTO_NETDEV) + if (table->family != NFPROTO_NETDEV && + table->family != NFPROTO_INET) continue; ctx.family = table->family; @@ -381,6 +383,11 @@ static int nf_tables_netdev_event(struct notifier_block *this, if (!nft_is_base_chain(chain)) continue; + basechain = nft_base_chain(chain); + if (table->family == NFPROTO_INET && + basechain->ops.hooknum != NF_INET_INGRESS) + continue; + ctx.chain = chain; nft_netdev_event(event, dev, &ctx); } diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index b8dbd20a6a4c5b705c1619115c358e60525c6bde..3f1e272efbb92cfc8b467f650ed8462ab118db4d 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -327,6 +327,12 @@ static int nft_target_validate(const struct nft_ctx *ctx, unsigned int hook_mask = 0; int ret; + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_BRIDGE && + ctx->family != NFPROTO_ARP) + return -EOPNOTSUPP; + if (nft_is_base_chain(ctx->chain)) { const struct nft_base_chain *basechain = nft_base_chain(ctx->chain); @@ -569,6 +575,12 @@ static int nft_match_validate(const struct nft_ctx *ctx, unsigned int hook_mask = 0; int ret; + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_BRIDGE && + ctx->family != NFPROTO_ARP) + return -EOPNOTSUPP; + if (nft_is_base_chain(ctx->chain)) { const struct nft_base_chain *basechain = nft_base_chain(ctx->chain); diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index a44340dd3ce649aedc600d9cbfd3e254460b7546..c2a5d05f501f7c5072f9e35c7e48a0d1c3472810 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -150,6 +150,11 @@ static int nft_flow_offload_validate(const struct nft_ctx *ctx, { unsigned int hook_mask = (1 << NF_INET_FORWARD); + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + return nft_chain_validate_hooks(ctx->chain, hook_mask); } diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index cd4eb4996aff32747859916c5eed1c72bc6da633..2e1ee7d9d9c3cd8d2b0f9f247945efbf38057e32 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -142,6 +142,11 @@ static int nft_nat_validate(const struct nft_ctx *ctx, struct nft_nat *priv = nft_expr_priv(expr); int err; + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + err = nft_chain_validate_dependency(ctx->chain, NFT_CHAIN_T_NAT); if (err < 0) return err; diff --git a/net/netfilter/nft_rt.c b/net/netfilter/nft_rt.c index bcd01a63e38f1bdb423cc9be816d3612beb533e6..f4a96164a5a1129484aec7ec6c4c8dda7ec894ad 100644 --- a/net/netfilter/nft_rt.c +++ b/net/netfilter/nft_rt.c @@ -166,6 +166,11 @@ static int nft_rt_validate(const struct nft_ctx *ctx, const struct nft_expr *exp const struct nft_rt *priv = nft_expr_priv(expr); unsigned int hooks; + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + switch (priv->key) { case NFT_RT_NEXTHOP4: case NFT_RT_NEXTHOP6: diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index f6d517185d9c0806ec17d499616e326c53b496b0..826e5f8c78f34c9c20585727a0b431fcd8211901 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -166,6 +166,11 @@ static int nft_socket_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nft_data **data) { + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_PRE_ROUTING) | (1 << NF_INET_LOCAL_IN) | diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 1133e06f3c40eb4cd4b1c13ed6a7e4dd6734f3c5..0806813d3a7673fe6306bf9a582b078bce40f571 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -186,7 +186,6 @@ static int nft_synproxy_do_init(const struct nft_ctx *ctx, break; #endif case NFPROTO_INET: - case NFPROTO_BRIDGE: err = nf_synproxy_ipv4_init(snet, ctx->net); if (err) goto nf_ct_failure; @@ -219,7 +218,6 @@ static void nft_synproxy_do_destroy(const struct nft_ctx *ctx) break; #endif case NFPROTO_INET: - case NFPROTO_BRIDGE: nf_synproxy_ipv4_fini(snet, ctx->net); nf_synproxy_ipv6_fini(snet, ctx->net); break; @@ -253,6 +251,11 @@ static int nft_synproxy_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nft_data **data) { + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_LOCAL_IN) | (1 << NF_INET_FORWARD)); } diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index f8d277e05ef4ff7359d8ffff69d9e17091598a82..6b606e83cdb6cd32d9c0674ca9b5c357a32745e8 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -293,6 +293,11 @@ static int nft_tproxy_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nft_data **data) { + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + return nft_chain_validate_hooks(ctx->chain, 1 << NF_INET_PRE_ROUTING); } diff --git a/net/netfilter/nft_xfrm.c b/net/netfilter/nft_xfrm.c index cbbbc4ecad3aed11b584db94f6bc6543ce09fe34..7f762fc428912ac16d1ffd7c38c67bd9c12d08e0 100644 --- a/net/netfilter/nft_xfrm.c +++ b/net/netfilter/nft_xfrm.c @@ -233,6 +233,11 @@ static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *e const struct nft_xfrm *priv = nft_expr_priv(expr); unsigned int hooks; + if (ctx->family != NFPROTO_IPV4 && + ctx->family != NFPROTO_IPV6 && + ctx->family != NFPROTO_INET) + return -EOPNOTSUPP; + switch (priv->dir) { case XFRM_POLICY_IN: hooks = (1 << NF_INET_FORWARD) | diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index fd5c7d76b0fcfb3ee0dce471c1ce3bba1cc603ed..a2b14434d7aa03d9c776510a40f472c7519798d6 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -366,7 +366,7 @@ static void netlink_skb_destructor(struct sk_buff *skb) if (is_vmalloc_addr(skb->head)) { if (!skb->cloned || !atomic_dec_return(&(skb_shinfo(skb)->dataref))) - vfree(skb->head); + vfree_atomic(skb->head); skb->head = NULL; } diff --git a/net/rxrpc/conn_service.c b/net/rxrpc/conn_service.c index 68508166bbc0b5c5b71a87e9af92a2fa13b6e6c5..af0e95ef992d0e51db176c25a905a4796e79aad7 100644 --- a/net/rxrpc/conn_service.c +++ b/net/rxrpc/conn_service.c @@ -31,7 +31,7 @@ struct rxrpc_connection *rxrpc_find_service_conn_rcu(struct rxrpc_peer *peer, struct rxrpc_conn_proto k; struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rb_node *p; - unsigned int seq = 0; + unsigned int seq = 1; k.epoch = sp->hdr.epoch; k.cid = sp->hdr.cid & RXRPC_CIDMASK; @@ -41,6 +41,7 @@ struct rxrpc_connection *rxrpc_find_service_conn_rcu(struct rxrpc_peer *peer, * under just the RCU read lock, so we have to check for * changes. */ + seq++; /* 2 on the 1st/lockless path, otherwise odd */ read_seqbegin_or_lock(&peer->service_conn_lock, &seq); p = rcu_dereference_raw(peer->service_conns.rb_node); diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c index f15fca59b4b2654251a375bc522d88a12b450ccc..7c921760dce785f3b0768d9c972a6e841dec63ba 100644 --- a/net/smc/smc_diag.c +++ b/net/smc/smc_diag.c @@ -177,7 +177,7 @@ static int __smc_diag_dump(struct sock *sk, struct sk_buff *skb, } if (smc->conn.lgr && smc->conn.lgr->is_smcd && (req->diag_ext & (1 << (SMC_DIAG_DMBINFO - 1))) && - !list_empty(&smc->conn.lgr->list)) { + !list_empty(&smc->conn.lgr->list) && smc->conn.rmb_desc) { struct smc_connection *conn = &smc->conn; struct smcd_diag_dmbinfo dinfo; diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 1e6dfe204ff3639beaace10dfe65adb68eff0a70..0c59d704ace2baac13f3dc8adf5ea083b8fe0870 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -2768,10 +2768,14 @@ int cfg80211_wext_siwscan(struct net_device *dev, wiphy = &rdev->wiphy; /* Determine number of channels, needed to allocate creq */ - if (wreq && wreq->num_channels) + if (wreq && wreq->num_channels) { + /* Passed from userspace so should be checked */ + if (unlikely(wreq->num_channels > IW_MAX_FREQUENCIES)) + return -EINVAL; n_channels = wreq->num_channels; - else + } else { n_channels = ieee80211_get_num_supported_channels(wiphy); + } creq = kzalloc(sizeof(*creq) + sizeof(struct cfg80211_ssid) + n_channels * sizeof(void *), diff --git a/scripts/Makefile b/scripts/Makefile index 9adb6d247818f1355e2e478256022cd10afd05f2..5d89c2ee274837d4f519dabae9271a2094b46cfb 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -24,10 +24,17 @@ HOSTCFLAGS_extract-cert.o = $(CRYPTO_CFLAGS) HOSTLDLIBS_extract-cert = $(CRYPTO_LIBS) ifdef CONFIG_UNWINDER_ORC +# Additional ARCH settings for x86 ifeq ($(ARCH),x86_64) ARCH := x86 endif -HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include + +# Additional ARCH settings for loongarch +ifeq ($(ARCH),loongarch64) +ARCH := loongarch +endif + +HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/$(ARCH)/include HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED HOSTLDLIBS_sorttable = -lpthread endif diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl index 92d9aa6cc4f5d2bd98c845adffb1e3b37d5e3749..db6098c42fa6b52aca5058834fab78d7e94db575 100755 --- a/scripts/get_abi.pl +++ b/scripts/get_abi.pl @@ -75,7 +75,7 @@ sub parse_abi { $name =~ s,.*/,,; my $fn = $file; - $fn =~ s,Documentation/ABI/,,; + $fn =~ s,.*Documentation/ABI/,,; my $nametag = "File $fn"; $data{$nametag}->{what} = "File $name"; diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 54ad86d13784995cfb8eb57bd0827204c6c36886..dd700c587f5d0390aaf21f4e8e2940d9fb00fe73 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -108,7 +108,8 @@ static bool is_ignored_symbol(const char *name, char type) /* Symbol names that begin with the following are ignored.*/ static const char * const ignored_prefixes[] = { "$", /* local symbols for ARM, MIPS, etc. */ - ".LASANPC", /* s390 kasan local symbols */ + "L0", /* LoongArch local symbols */ + ".L", /* LoongArch/S390 local symbols */ "__crc_", /* modversions */ "__efistub_", /* arm64 EFI stub namespace */ "__kvm_nvhe_", /* arm64 non-VHE KVM namespace */ diff --git a/scripts/kconfig/lexer.l b/scripts/kconfig/lexer.l index b5a1d31458f928628d0548dd48e1d71a59a65a67..9f62ebaa3ea69c112c695f946cf4bbc466a032a1 100644 --- a/scripts/kconfig/lexer.l +++ b/scripts/kconfig/lexer.l @@ -30,6 +30,7 @@ static const char *kconfig_white_list[] = { "drivers/staging/ucollection/Kconfig", "fs/proc/memory_security/Kconfig", "fs/code_sign/Kconfig", + "fs/dec/Kconfig", "security/container_escape_detection/Kconfig", }; diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index cde1ce6f171b5ea87de6b825c053eb219c1b06bf..aa43a8d13f532b853cd263e9c1c5792ec7ce634d 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1852,6 +1852,14 @@ static int addend_mips_rel(struct elf_info *elf, Elf_Shdr *sechdr, Elf_Rela *r) return 0; } +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 +#endif + +#ifndef R_LARCH_SUB32 +#define R_LARCH_SUB32 55 +#endif + static void section_rela(const char *modname, struct elf_info *elf, Elf_Shdr *sechdr) { @@ -1888,6 +1896,13 @@ static void section_rela(const char *modname, struct elf_info *elf, r_sym = ELF_R_SYM(r.r_info); #endif r.r_addend = TO_NATIVE(rela->r_addend); + switch (elf->hdr->e_machine) { + case EM_LOONGARCH: + if (!strcmp("__ex_table", fromsec) && + ELF_R_TYPE(r.r_info) == R_LARCH_SUB32) + continue; + break; + } sym = elf->symtab_start + r_sym; /* Skip special sections */ if (is_shndx_special(sym->st_shndx)) diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index ec692af8ce9ebe3508a4d16c9b4e92a98a650fa7..90f1d9768c16b6a2654f534252b953f4152ecb09 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -38,6 +38,14 @@ #define R_AARCH64_ABS64 257 #endif +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 +#define R_LARCH_32 1 +#define R_LARCH_64 2 +#define R_LARCH_MARK_LA 20 +#define R_LARCH_SOP_PUSH_PLT_PCREL 29 +#endif + #define R_ARM_PC24 1 #define R_ARM_THM_CALL 10 #define R_ARM_CALL 28 @@ -445,6 +453,28 @@ static int arm64_is_fake_mcount(Elf64_Rel const *rp) return ELF64_R_TYPE(w8(rp->r_info)) != R_AARCH64_CALL26; } +static int LARCH32_is_fake_mcount(Elf32_Rel const *rp) +{ + switch (ELF64_R_TYPE(w(rp->r_info))) { + case R_LARCH_MARK_LA: + case R_LARCH_SOP_PUSH_PLT_PCREL: + return 0; + } + + return 1; +} + +static int LARCH64_is_fake_mcount(Elf64_Rel const *rp) +{ + switch (ELF64_R_TYPE(w(rp->r_info))) { + case R_LARCH_MARK_LA: + case R_LARCH_SOP_PUSH_PLT_PCREL: + return 0; + } + + return 1; +} + /* 64-bit EM_MIPS has weird ELF64_Rela.r_info. * http://techpubs.sgi.com/library/manuals/4000/007-4658-001/pdf/007-4658-001.pdf * We interpret Table 29 Relocation Operation (Elf64_Rel, Elf64_Rela) [p.40] @@ -562,6 +592,7 @@ static int do_file(char const *const fname) break; case EM_IA_64: reltype = R_IA64_IMM64; break; case EM_MIPS: /* reltype: e_class */ break; + case EM_LOONGARCH: /* reltype: e_class */ break; case EM_PPC: reltype = R_PPC_ADDR32; break; case EM_PPC64: reltype = R_PPC64_ADDR64; break; case EM_S390: /* reltype: e_class */ break; @@ -593,6 +624,10 @@ static int do_file(char const *const fname) reltype = R_MIPS_32; is_fake_mcount32 = MIPS32_is_fake_mcount; } + if (w2(ehdr->e_machine) == EM_LOONGARCH) { + reltype = R_LARCH_32; + is_fake_mcount32 = LARCH32_is_fake_mcount; + } if (do32(ehdr, fname, reltype) < 0) goto out; break; @@ -614,6 +649,10 @@ static int do_file(char const *const fname) Elf64_r_info = MIPS64_r_info; is_fake_mcount64 = MIPS64_is_fake_mcount; } + if (w2(ghdr->e_machine) == EM_LOONGARCH) { + reltype = R_LARCH_64; + is_fake_mcount64 = LARCH64_is_fake_mcount; + } if (do64(ghdr, fname, reltype) < 0) goto out; break; diff --git a/scripts/sorttable.c b/scripts/sorttable.c index ac93e033b7cbcd07da300b504ef4a5f4d0aec0eb..84044a04518875ea104a1e94b0ddf37373ef715f 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -54,6 +54,10 @@ #define EM_ARCV2 195 #endif +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 +#endif + static uint32_t (*r)(const uint32_t *); static uint16_t (*r2)(const uint16_t *); static uint64_t (*r8)(const uint64_t *); @@ -227,6 +231,34 @@ static void sort_relative_table(char *extab_image, int image_size) } } +static void sort_relative_table_with_data(char *extab_image, int image_size) +{ + int i = 0; + + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + + w(r(loc) + i, loc); + w(r(loc + 1) + i + 4, loc + 1); + /* Don't touch the fixup type or data */ + + i += sizeof(uint32_t) * 3; + } + + qsort(extab_image, image_size / 12, 12, compare_relative_table); + + i = 0; + while (i < image_size) { + uint32_t *loc = (uint32_t *)(extab_image + i); + + w(r(loc) - i, loc); + w(r(loc + 1) - (i + 4), loc + 1); + /* Don't touch the fixup type or data */ + + i += sizeof(uint32_t) * 3; + } +} + static void x86_sort_relative_table(char *extab_image, int image_size) { int i = 0; @@ -338,6 +370,9 @@ static int do_file(char const *const fname, void *addr) case EM_S390: custom_sort = s390_sort_relative_table; break; + case EM_LOONGARCH: + custom_sort = sort_relative_table_with_data; + break; case EM_AARCH64: case EM_PARISC: case EM_PPC: diff --git a/scripts/subarch.include b/scripts/subarch.include index 650682821126c39fb145b924b8340a07eeec1bd7..c79e0d0b1a19e7ea6bc7b0f45f28bc473a6e7a0f 100644 --- a/scripts/subarch.include +++ b/scripts/subarch.include @@ -10,4 +10,4 @@ SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ -e s/s390x/s390/ -e s/parisc64/parisc/ \ -e s/ppc.*/powerpc/ -e s/mips.*/mips/ \ -e s/sh[234].*/sh/ -e s/aarch64.*/arm64/ \ - -e s/riscv.*/riscv/) + -e s/riscv.*/riscv/ -e s/loongarch.*/loongarch/) diff --git a/security/security.c b/security/security.c index 60e3bee786e52e1aa35ef8dfea8d4ddc12ffd971..e0634a7090164704df060f3b5e2169cfbba34d59 100644 --- a/security/security.c +++ b/security/security.c @@ -1498,6 +1498,24 @@ int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } EXPORT_SYMBOL_GPL(security_file_ioctl); +/** + * security_file_ioctl_compat() - Check if an ioctl is allowed in compat mode + * @file: associated file + * @cmd: ioctl cmd + * @arg: ioctl arguments + * + * Compat version of security_file_ioctl() that correctly handles 32-bit + * processes running on 64-bit kernels. + * + * Return: Returns 0 if permission is granted. + */ +int security_file_ioctl_compat(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return call_int_hook(file_ioctl_compat, 0, file, cmd, arg); +} +EXPORT_SYMBOL_GPL(security_file_ioctl_compat); + static inline unsigned long mmap_prot(struct file *file, unsigned long prot) { /* diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 86470a7b7e17925f8b2d37f11a4649146a37f78e..725b1d707d76e1a6be441fda2017fd9e67651758 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3664,6 +3664,33 @@ static int selinux_file_ioctl(struct file *file, unsigned int cmd, return error; } +static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd, + unsigned long arg) +{ + /* + * If we are in a 64-bit kernel running 32-bit userspace, we need to + * make sure we don't compare 32-bit flags to 64-bit flags. + */ + switch (cmd) { + case FS_IOC32_GETFLAGS: + cmd = FS_IOC_GETFLAGS; + break; + case FS_IOC32_SETFLAGS: + cmd = FS_IOC_SETFLAGS; + break; + case FS_IOC32_GETVERSION: + cmd = FS_IOC_GETVERSION; + break; + case FS_IOC32_SETVERSION: + cmd = FS_IOC_SETVERSION; + break; + default: + break; + } + + return selinux_file_ioctl(file, cmd, arg); +} + static int default_noexec __ro_after_init; static int file_map_prot_check(struct file *file, unsigned long prot, int shared) @@ -7052,6 +7079,7 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(file_permission, selinux_file_permission), LSM_HOOK_INIT(file_alloc_security, selinux_file_alloc_security), LSM_HOOK_INIT(file_ioctl, selinux_file_ioctl), + LSM_HOOK_INIT(file_ioctl_compat, selinux_file_ioctl_compat), LSM_HOOK_INIT(mmap_file, selinux_mmap_file), LSM_HOOK_INIT(mmap_addr, selinux_mmap_addr), LSM_HOOK_INIT(file_mprotect, selinux_file_mprotect), diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 814518ad4402b9abebf260d014749285eff477cd..e1669759403a6f0ddbb16bf1c562b260644019a6 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -4767,6 +4767,7 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(file_alloc_security, smack_file_alloc_security), LSM_HOOK_INIT(file_ioctl, smack_file_ioctl), + LSM_HOOK_INIT(file_ioctl_compat, smack_file_ioctl), LSM_HOOK_INIT(file_lock, smack_file_lock), LSM_HOOK_INIT(file_fcntl, smack_file_fcntl), LSM_HOOK_INIT(mmap_file, smack_mmap_file), diff --git a/security/tomoyo/tomoyo.c b/security/tomoyo/tomoyo.c index 1f3cd432d830875d47e30d83d96ba4f2cd07b3af..a8dc3ae938f9c222212e9540a9b9862ec767007b 100644 --- a/security/tomoyo/tomoyo.c +++ b/security/tomoyo/tomoyo.c @@ -548,6 +548,7 @@ static struct security_hook_list tomoyo_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(path_rename, tomoyo_path_rename), LSM_HOOK_INIT(inode_getattr, tomoyo_inode_getattr), LSM_HOOK_INIT(file_ioctl, tomoyo_file_ioctl), + LSM_HOOK_INIT(file_ioctl_compat, tomoyo_file_ioctl), LSM_HOOK_INIT(path_chmod, tomoyo_path_chmod), LSM_HOOK_INIT(path_chown, tomoyo_path_chown), LSM_HOOK_INIT(path_chroot, tomoyo_path_chroot), diff --git a/sound/hda/hdac_bus.c b/sound/hda/hdac_bus.c index 9766f6af87430db6068b812c7308f2767d8ad30b..6d4c780cf818d3fa7d55bc63389c6a0d03f100e7 100644 --- a/sound/hda/hdac_bus.c +++ b/sound/hda/hdac_bus.c @@ -11,6 +11,7 @@ #include #include "local.h" #include "trace.h" +#include "../pci/hda/hda_controller.h" static void snd_hdac_bus_process_unsol_events(struct work_struct *work); @@ -108,6 +109,7 @@ int snd_hdac_bus_exec_verb_unlocked(struct hdac_bus *bus, unsigned int addr, { unsigned int tmp; int err; + struct azx *chip = bus_to_azx(bus); if (cmd == ~0) return -EINVAL; @@ -116,7 +118,9 @@ int snd_hdac_bus_exec_verb_unlocked(struct hdac_bus *bus, unsigned int addr, *res = -1; else if (bus->sync_write) res = &tmp; - for (;;) { + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + err = bus->ops->command(bus, cmd); + else for (;;) { trace_hda_send_cmd(bus, cmd); err = bus->ops->command(bus, cmd); if (err != -EAGAIN) diff --git a/sound/hda/hdac_controller.c b/sound/hda/hdac_controller.c index 522d1897659cbd032421257662ece16e201fd53e..da047c8bb239335ae9b29a6ebe6686f0c9dd6eb7 100644 --- a/sound/hda/hdac_controller.c +++ b/sound/hda/hdac_controller.c @@ -10,6 +10,7 @@ #include #include #include "local.h" +#include "../pci/hda/hda_controller.h" /* clear CORB read pointer properly */ static void azx_clear_corbrp(struct hdac_bus *bus) @@ -42,6 +43,8 @@ static void azx_clear_corbrp(struct hdac_bus *bus) */ void snd_hdac_bus_init_cmd_io(struct hdac_bus *bus) { + struct azx *chip = bus_to_azx(bus); + WARN_ON_ONCE(!bus->rb.area); spin_lock_irq(&bus->reg_lock); @@ -58,11 +61,15 @@ void snd_hdac_bus_init_cmd_io(struct hdac_bus *bus) /* reset the corb hw read pointer */ snd_hdac_chip_writew(bus, CORBRP, AZX_CORBRP_RST); - if (!bus->corbrp_self_clear) + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_chip_writew(bus, CORBRP, 0); + else if (!bus->corbrp_self_clear) azx_clear_corbrp(bus); /* enable corb dma */ snd_hdac_chip_writeb(bus, CORBCTL, AZX_CORBCTL_RUN); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_chip_readb(bus, CORBCTL); /* RIRB set up */ bus->rirb.addr = bus->rb.addr + 2048; @@ -79,7 +86,12 @@ void snd_hdac_bus_init_cmd_io(struct hdac_bus *bus) /* set N=1, get RIRB response interrupt for new entry */ snd_hdac_chip_writew(bus, RINTCNT, 1); /* enable rirb dma and response irq */ - snd_hdac_chip_writeb(bus, RIRBCTL, AZX_RBCTL_DMA_EN | AZX_RBCTL_IRQ_EN); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + snd_hdac_chip_writeb(bus, RIRBCTL, AZX_RBCTL_DMA_EN); + snd_hdac_chip_readb(bus, RIRBCTL); + } + else + snd_hdac_chip_writeb(bus, RIRBCTL, AZX_RBCTL_DMA_EN | AZX_RBCTL_IRQ_EN); /* Accept unsolicited responses */ snd_hdac_chip_updatel(bus, GCTL, AZX_GCTL_UNSOL, AZX_GCTL_UNSOL); spin_unlock_irq(&bus->reg_lock); @@ -132,6 +144,18 @@ static unsigned int azx_command_addr(u32 cmd) return addr; } +static unsigned int azx_response_addr(u32 res) +{ + unsigned int addr = res & 0xf; + + if (addr >= AZX_MAX_CODECS) { + snd_BUG(); + addr = 0; + } + + return addr; +} + /** * snd_hdac_bus_send_cmd - send a command verb via CORB * @bus: HD-audio core bus @@ -207,13 +231,8 @@ void snd_hdac_bus_update_rirb(struct hdac_bus *bus) rp = bus->rirb.rp << 1; /* an RIRB entry is 8-bytes */ res_ex = le32_to_cpu(bus->rirb.buf[rp + 1]); res = le32_to_cpu(bus->rirb.buf[rp]); - addr = res_ex & 0xf; - if (addr >= HDA_MAX_CODECS) { - dev_err(bus->dev, - "spurious response %#x:%#x, rp = %d, wp = %d", - res, res_ex, bus->rirb.rp, wp); - snd_BUG(); - } else if (res_ex & AZX_RIRB_EX_UNSOL_EV) + addr = azx_response_addr(res_ex); + if (res_ex & AZX_RIRB_EX_UNSOL_EV) snd_hdac_bus_queue_event(bus, res, res_ex); else if (bus->rirb.cmds[addr]) { bus->rirb.res[addr] = res; @@ -485,16 +504,24 @@ static void azx_int_disable(struct hdac_bus *bus) static void azx_int_clear(struct hdac_bus *bus) { struct hdac_stream *azx_dev; + struct azx *chip = bus_to_azx(bus); /* clear stream status */ - list_for_each_entry(azx_dev, &bus->stream_list, list) - snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); + list_for_each_entry(azx_dev, &bus->stream_list, list) { + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_stream_updateb(azx_dev, SD_STS, 0, 0); + else + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); + } /* clear STATESTS */ snd_hdac_chip_writew(bus, STATESTS, STATESTS_INT_MASK); /* clear rirb status */ - snd_hdac_chip_writeb(bus, RIRBSTS, RIRB_INT_MASK); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_chip_updateb(bus, RIRBSTS, ~RIRB_INT_MASK, 0); + else + snd_hdac_chip_writeb(bus, RIRBSTS, RIRB_INT_MASK); /* clear int status */ snd_hdac_chip_writel(bus, INTSTS, AZX_INT_CTRL_EN | AZX_INT_ALL_STREAM); @@ -575,11 +602,17 @@ int snd_hdac_bus_handle_stream_irq(struct hdac_bus *bus, unsigned int status, struct hdac_stream *azx_dev; u8 sd_status; int handled = 0; + struct azx *chip = bus_to_azx(bus); list_for_each_entry(azx_dev, &bus->stream_list, list) { if (status & azx_dev->sd_int_sta_mask) { sd_status = snd_hdac_stream_readb(azx_dev, SD_STS); - snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + snd_hdac_stream_writeb(azx_dev, SD_STS, sd_status); + snd_hdac_stream_readb(azx_dev, SD_STS); + } + else + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); handled |= 1 << azx_dev->index; if (!azx_dev->substream || !azx_dev->running || !(sd_status & SD_INT_COMPLETE)) diff --git a/sound/hda/hdac_device.c b/sound/hda/hdac_device.c index bfd85857767671adfe9d61be9a196dab709cafd2..5315a3eb114eeb4af46406a911c1e60c126108f3 100644 --- a/sound/hda/hdac_device.c +++ b/sound/hda/hdac_device.c @@ -645,6 +645,7 @@ struct hda_vendor_id { }; static const struct hda_vendor_id hda_vendor_ids[] = { + { 0x0014, "Loongson" }, { 0x1002, "ATI" }, { 0x1013, "Cirrus Logic" }, { 0x1057, "Motorola" }, diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c index 5570722458caf54ea63325fc216bd84c44e82c88..ae6712b8df48e1bf010618495a7b1bc5dc792293 100644 --- a/sound/hda/hdac_stream.c +++ b/sound/hda/hdac_stream.c @@ -12,6 +12,7 @@ #include #include #include "trace.h" +#include "../pci/hda/hda_controller.h" /** * snd_hdac_get_stream_stripe_ctl - get stripe control value @@ -83,12 +84,13 @@ EXPORT_SYMBOL_GPL(snd_hdac_stream_init); void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start) { struct hdac_bus *bus = azx_dev->bus; + struct azx *chip = bus_to_azx(bus); int stripe_ctl; trace_snd_hdac_stream_start(bus, azx_dev); azx_dev->start_wallclk = snd_hdac_chip_readl(bus, WALLCLK); - if (!fresh_start) + if (!fresh_start && !(chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND)) azx_dev->start_wallclk -= azx_dev->period_wallclk; /* enable SIE */ @@ -101,11 +103,19 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start) stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream); else stripe_ctl = 0; - snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, + stripe_ctl); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, stripe_ctl); } /* set DMA start and interrupt mask */ - snd_hdac_stream_updateb(azx_dev, SD_CTL, + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + snd_hdac_stream_updatel(azx_dev, SD_CTL, + 0, SD_CTL_DMA_START | SD_INT_MASK); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_DMA_START | SD_INT_MASK); azx_dev->running = true; } @@ -117,11 +127,31 @@ EXPORT_SYMBOL_GPL(snd_hdac_stream_start); */ void snd_hdac_stream_clear(struct hdac_stream *azx_dev) { - snd_hdac_stream_updateb(azx_dev, SD_CTL, - SD_CTL_DMA_START | SD_INT_MASK, 0); - snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ - if (azx_dev->stripe) - snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + int stream; + struct azx *chip = bus_to_azx(azx_dev->bus); + struct snd_pcm_substream *substream = azx_dev->substream; + + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + snd_hdac_stream_updatel(azx_dev, SD_CTL, + SD_CTL_DMA_START | SD_INT_MASK, 0); + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ + if (azx_dev->stripe) + snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + + if (!substream) + stream_to_azx_dev(azx_dev)->fix_prvpos = 0; + else { + stream = substream->stream; + stream_to_azx_dev(azx_dev)->fix_prvpos = + chip->get_position[stream](chip, stream_to_azx_dev(azx_dev)); + } + } else { + snd_hdac_stream_updateb(azx_dev, SD_CTL, + SD_CTL_DMA_START | SD_INT_MASK, 0); + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ + if (azx_dev->stripe) + snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + } azx_dev->running = false; } EXPORT_SYMBOL_GPL(snd_hdac_stream_clear); @@ -178,6 +208,10 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev) unsigned char val; int timeout; int dma_run_state; + struct azx *chip = bus_to_azx(azx_dev->bus); + + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + goto out; snd_hdac_stream_clear(azx_dev); @@ -209,6 +243,7 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev) break; } while (--timeout); +out: /* reset first position - may not be synced with hw at this time */ if (azx_dev->posbuf) *azx_dev->posbuf = 0; @@ -224,6 +259,7 @@ int snd_hdac_stream_setup(struct hdac_stream *azx_dev) struct hdac_bus *bus = azx_dev->bus; struct snd_pcm_runtime *runtime; unsigned int val; + struct azx *chip = bus_to_azx(bus); if (azx_dev->substream) runtime = azx_dev->substream->runtime; @@ -240,7 +276,14 @@ int snd_hdac_stream_setup(struct hdac_stream *azx_dev) snd_hdac_stream_writel(azx_dev, SD_CTL, val); /* program the length of samples in cyclic buffer */ - snd_hdac_stream_writel(azx_dev, SD_CBL, azx_dev->bufsize); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + if(azx_dev->substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + snd_hdac_stream_writel(azx_dev, SD_CBL, azx_dev->bufsize - 64); + else + snd_hdac_stream_writel(azx_dev, SD_CBL, azx_dev->bufsize - 16); + } + else + snd_hdac_stream_writel(azx_dev, SD_CBL, azx_dev->bufsize); /* program the stream format */ /* this value needs to be the same as the one programmed */ @@ -448,6 +491,7 @@ int snd_hdac_stream_setup_periods(struct hdac_stream *azx_dev) __le32 *bdl; int i, ofs, periods, period_bytes; int pos_adj, pos_align; + struct azx *chip = bus_to_azx(bus); /* reset BDL address */ snd_hdac_stream_writel(azx_dev, SD_BDLPL, 0); @@ -462,6 +506,8 @@ int snd_hdac_stream_setup_periods(struct hdac_stream *azx_dev) azx_dev->frags = 0; pos_adj = bus->bdl_pos_adj; + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + pos_adj = 0; if (!azx_dev->no_period_wakeup && pos_adj > 0) { pos_align = pos_adj; pos_adj = (pos_adj * runtime->rate + 47999) / 48000; diff --git a/sound/pci/hda/Kconfig b/sound/pci/hda/Kconfig index 90759391cbac826b81bf5d5242b92dc0a3aaeeb5..77ee0b08805060a2be2b204f64e44d5b7af5e80f 100644 --- a/sound/pci/hda/Kconfig +++ b/sound/pci/hda/Kconfig @@ -42,6 +42,17 @@ config SND_HDA_TEGRA To compile this driver as a module, choose M here: the module will be called snd-hda-tegra. +config SND_HDA_LOONGSON + tristate "Loongson HD Audio" + depends on MACH_LOONGSON64 + select SND_HDA + help + Say Y here to include support for Loongson's LS7A "High + Definition Audio" controller. + + To compile this driver as a module, choose M here: the module + will be called snd-hda-loongson. + if SND_HDA config SND_HDA_HWDEP diff --git a/sound/pci/hda/Makefile b/sound/pci/hda/Makefile index b57432f000568eb1b40f5131464218ba1f07bcff..2b6d1b32409c339993781c56f68e83302fa74795 100644 --- a/sound/pci/hda/Makefile +++ b/sound/pci/hda/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 snd-hda-intel-objs := hda_intel.o snd-hda-tegra-objs := hda_tegra.o +snd-hda-loongson-objs := hda_loongson.o snd-hda-codec-y := hda_bind.o hda_codec.o hda_jack.o hda_auto_parser.o hda_sysfs.o snd-hda-codec-y += hda_controller.o @@ -48,3 +49,4 @@ obj-$(CONFIG_SND_HDA_CODEC_HDMI) += snd-hda-codec-hdmi.o # when built in kernel obj-$(CONFIG_SND_HDA_INTEL) += snd-hda-intel.o obj-$(CONFIG_SND_HDA_TEGRA) += snd-hda-tegra.o +obj-$(CONFIG_SND_HDA_LOONGSON) += snd-hda-loongson.o diff --git a/sound/pci/hda/hda_controller.c b/sound/pci/hda/hda_controller.c index ea76395d71d386950c24e03a82036404d95a38c8..02279958a74a52c8bafa6c4548df450b421b5e7c 100644 --- a/sound/pci/hda/hda_controller.c +++ b/sound/pci/hda/hda_controller.c @@ -295,6 +295,23 @@ unsigned int azx_get_position(struct azx *chip, int stream = substream->stream; int delay = 0; + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + pos = chip->get_position[stream](chip, azx_dev); + + if (pos >= azx_dev->fix_prvpos) { + pos = pos - azx_dev->fix_prvpos; + pos %= azx_dev->core.bufsize; + } else { + if (azx_dev->fix_prvpos > azx_dev->core.bufsize) + pos = (0x100000000ULL + pos-azx_dev->fix_prvpos) + % azx_dev->core.bufsize; + else + pos = pos + azx_dev->core.bufsize - azx_dev->fix_prvpos; + } + + return pos; + } + if (chip->get_position[stream]) pos = chip->get_position[stream](chip, azx_dev); else /* use the position buffer as default */ @@ -920,6 +937,8 @@ static int azx_send_cmd(struct hdac_bus *bus, unsigned int val) if (chip->disabled) return 0; + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + udelay(500); if (chip->single_cmd) return azx_single_send_cmd(bus, val); else @@ -1076,8 +1095,9 @@ static void stream_update(struct hdac_bus *bus, struct hdac_stream *s) irqreturn_t azx_interrupt(int irq, void *dev_id) { struct azx *chip = dev_id; + struct hdac_stream *azx_dev; struct hdac_bus *bus = azx_bus(chip); - u32 status; + u32 i = 0, status = 0; bool active, handled = false; int repeat = 0; /* count for avoiding endless loop */ @@ -1093,8 +1113,21 @@ irqreturn_t azx_interrupt(int irq, void *dev_id) goto unlock; do { - status = azx_readl(chip, INTSTS); - if (status == 0 || status == 0xffffffff) + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) { + i = 0; + status = 0; + list_for_each_entry(azx_dev, &bus->stream_list, list) { + status |= (snd_hdac_stream_readb(azx_dev, SD_STS) & SD_INT_MASK) ? + (1 << i) : 0; + i++; + } + status |= (status & ~0) ? (1 << 31) : 0; + } + else + status = azx_readl(chip, INTSTS); + + if (status == 0 || + (status == 0xffffffff && !(chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND))) break; handled = true; @@ -1112,7 +1145,10 @@ irqreturn_t azx_interrupt(int irq, void *dev_id) * remain unserviced by IRQ, eventually falling back * to polling mode in azx_rirb_get_response. */ - azx_writeb(chip, RIRBSTS, RIRB_INT_MASK); + if (chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND) + azx_writeb(chip, RIRBSTS, status & RIRB_INT_MASK); + else + azx_writeb(chip, RIRBSTS, RIRB_INT_MASK); active = true; if (status & RIRB_INT_RESPONSE) { if (chip->driver_caps & AZX_DCAPS_CTX_WORKAROUND) diff --git a/sound/pci/hda/hda_controller.h b/sound/pci/hda/hda_controller.h index 324cba13c7bacd2c4349ef18e50ddbeb5422bc32..c5cb3e5fd89a49579c9622cdc8f319e20c6f08b6 100644 --- a/sound/pci/hda/hda_controller.h +++ b/sound/pci/hda/hda_controller.h @@ -45,6 +45,7 @@ #define AZX_DCAPS_CORBRP_SELF_CLEAR (1 << 28) /* CORBRP clears itself after reset */ #define AZX_DCAPS_NO_MSI64 (1 << 29) /* Stick to 32-bit MSIs */ #define AZX_DCAPS_SEPARATE_STREAM_TAG (1 << 30) /* capture and playback use separate stream tag */ +#define AZX_DCAPS_LS2X_WORKAROUND (1 << 31) /* Loongson-2H workaround */ enum { AZX_SNOOP_TYPE_NONE, @@ -63,6 +64,9 @@ struct azx_dev { * when link position is not greater than FIFO size */ unsigned int insufficient:1; + + /* For Loongson */ + unsigned int fix_prvpos; }; #define azx_stream(dev) (&(dev)->core) diff --git a/sound/pci/hda/hda_loongson.c b/sound/pci/hda/hda_loongson.c new file mode 100644 index 0000000000000000000000000000000000000000..eafde6cc74ecf08d3af51e304fd931e5c260fe78 --- /dev/null +++ b/sound/pci/hda/hda_loongson.c @@ -0,0 +1,850 @@ +/* + * + * hda_loongson.c - Implementation of primary alsa driver code base + * for Intel HD Audio. + * + * Copyright (c) 2004 Intel Corporation. All rights reserved. + * + * Copyright (c) 2004 Takashi Iwai + * PeiSen Hou + * + * Copyright (c) 2014 Huacai Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * CONTACTS: + * + * Matt Jared matt.jared@intel.com + * Andy Kopp andy.kopp@intel.com + * Dan Kogan dan.d.kogan@intel.com + * + * CHANGES: + * + * 2004.12.01 Major rewrite by tiwai, merged the work of pshou + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "hda_controller.h" + +#define LS7A_NUM_CAPTURE 4 +#define LS7A_NUM_PLAYBACK 4 + +static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX; +static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR; +static bool enable[SNDRV_CARDS] = SNDRV_DEFAULT_ENABLE_PNP; +static char *model[SNDRV_CARDS]; +static int bdl_pos_adj[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS-1)] = -1}; +static int probe_only[SNDRV_CARDS]; +static int jackpoll_ms[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS-1)] = 1200}; +static bool single_cmd; +static int enable_msi = -1; +#ifdef CONFIG_SND_HDA_INPUT_BEEP +static bool beep_mode[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS-1)] = + CONFIG_SND_HDA_INPUT_BEEP_MODE}; +#endif + +module_param_array(index, int, NULL, 0444); +MODULE_PARM_DESC(index, "Index value for Intel HD audio interface."); +module_param_array(id, charp, NULL, 0444); +MODULE_PARM_DESC(id, "ID string for Intel HD audio interface."); +module_param_array(enable, bool, NULL, 0444); +MODULE_PARM_DESC(enable, "Enable Intel HD audio interface."); +module_param_array(model, charp, NULL, 0444); +MODULE_PARM_DESC(model, "Use the given board model."); +module_param_array(bdl_pos_adj, int, NULL, 0644); +MODULE_PARM_DESC(bdl_pos_adj, "BDL position adjustment offset."); +module_param_array(probe_only, int, NULL, 0444); +MODULE_PARM_DESC(probe_only, "Only probing and no codec initialization."); +module_param_array(jackpoll_ms, int, NULL, 0444); +MODULE_PARM_DESC(jackpoll_ms, "Ms between polling for jack events (default = 0, using unsol events only)"); +module_param(single_cmd, bool, 0444); +MODULE_PARM_DESC(single_cmd, "Use single command to communicate with codecs " + "(for debugging only)."); +module_param(enable_msi, bint, 0444); +MODULE_PARM_DESC(enable_msi, "Enable Message Signaled Interrupt (MSI)"); +#ifdef CONFIG_SND_HDA_INPUT_BEEP +module_param_array(beep_mode, bool, NULL, 0444); +MODULE_PARM_DESC(beep_mode, "Select HDA Beep registration mode " + "(0=off, 1=on) (default=1)."); +#endif + +#ifdef CONFIG_PM +static int param_set_xint(const char *val, const struct kernel_param *kp); +static struct kernel_param_ops param_ops_xint = { + .set = param_set_xint, + .get = param_get_int, +}; +#define param_check_xint param_check_int + +static int power_save = CONFIG_SND_HDA_POWER_SAVE_DEFAULT; +module_param(power_save, xint, 0644); +MODULE_PARM_DESC(power_save, "Automatic power-saving timeout " + "(in second, 0 = disable)."); + +/* reset the HD-audio controller in power save mode. + * this may give more power-saving, but will take longer time to + * wake up. + */ +static bool power_save_controller = 1; +module_param(power_save_controller, bool, 0644); +MODULE_PARM_DESC(power_save_controller, "Reset controller in power save mode."); +#else +#define power_save 0 +#endif /* CONFIG_PM */ + +static int align_buffer_size = -1; +module_param(align_buffer_size, bint, 0644); +MODULE_PARM_DESC(align_buffer_size, + "Force buffer and period sizes to be multiple of 128 bytes."); + +MODULE_LICENSE("GPL"); +MODULE_SUPPORTED_DEVICE("{Loongson, LS7A}"); +MODULE_DESCRIPTION("Loongson LS7A HDA driver"); + +/* driver types */ +enum { + AZX_DRIVER_LS7A, + AZX_DRIVER_HDMI, + AZX_NUM_DRIVERS, /* keep this as last entry */ +}; + +static char *driver_short_names[] = { + [AZX_DRIVER_LS7A] = "HD-Audio Loongson", + [AZX_DRIVER_HDMI] = "HD-Audio Loongson HDMI", +}; + +struct hda_loongson { + struct azx chip; + + /* for pending irqs */ + struct work_struct irq_pending_work; + + /* sync probing */ + struct completion probe_wait; + struct work_struct probe_work; + + /* card list (for power_save trigger) */ + struct list_head list; + + /* extra flags */ + unsigned int irq_pending_warned:1; + unsigned int probe_continued:1; +}; + +static int azx_acquire_irq(struct azx *chip, int do_disconnect); + +static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev); + +/* called from IRQ */ +static int azx_position_check(struct azx *chip, struct azx_dev *azx_dev) +{ + struct hda_loongson *hda = container_of(chip, struct hda_loongson, chip); + int ok; + + ok = azx_position_ok(chip, azx_dev); + if (ok == 1) { + azx_dev->irq_pending = 0; + return ok; + } else if (ok == 0) { + /* bogus IRQ, process it later */ + azx_dev->irq_pending = 1; + schedule_work(&hda->irq_pending_work); + } + return 0; +} + +/* + * Check whether the current DMA position is acceptable for updating + * periods. Returns non-zero if it's OK. + * + * Many HD-audio controllers appear pretty inaccurate about + * the update-IRQ timing. The IRQ is issued before actually the + * data is processed. So, we need to process it afterwords in a + * workqueue. + */ +static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev) +{ + return 1; /* OK, it's fine */ +} + +/* + * The work for pending PCM period updates. + */ +static void azx_irq_pending_work(struct work_struct *work) +{ + struct hda_loongson *hda = container_of(work, struct hda_loongson, irq_pending_work); + struct azx *chip = &hda->chip; + struct hdac_bus *bus = azx_bus(chip); + struct hdac_stream *s; + int pending, ok; + + if (!hda->irq_pending_warned) { + dev_info(chip->card->dev, + "IRQ timing workaround is activated for card #%d. Suggest a bigger bdl_pos_adj.\n", + chip->card->number); + hda->irq_pending_warned = 1; + } + + for (;;) { + pending = 0; + spin_lock_irq(&bus->reg_lock); + list_for_each_entry(s, &bus->stream_list, list) { + struct azx_dev *azx_dev = stream_to_azx_dev(s); + if (!azx_dev->irq_pending || + !s->substream || + !s->running) + continue; + ok = azx_position_ok(chip, azx_dev); + if (ok > 0) { + azx_dev->irq_pending = 0; + spin_unlock(&bus->reg_lock); + snd_pcm_period_elapsed(s->substream); + spin_lock(&bus->reg_lock); + } else if (ok < 0) { + pending = 0; /* too early */ + } else + pending++; + } + spin_unlock_irq(&bus->reg_lock); + if (!pending) + return; + msleep(1); + } +} + +/* clear irq_pending flags and assure no on-going workq */ +static void azx_clear_irq_pending(struct azx *chip) +{ + struct hdac_bus *bus = azx_bus(chip); + struct hdac_stream *s; + + spin_lock_irq(&bus->reg_lock); + list_for_each_entry(s, &bus->stream_list, list) { + struct azx_dev *azx_dev = stream_to_azx_dev(s); + azx_dev->irq_pending = 0; + } + spin_unlock_irq(&bus->reg_lock); +} + +static int azx_acquire_irq(struct azx *chip, int do_disconnect) +{ + struct hdac_bus *bus = azx_bus(chip); + int irq; + + if (chip->pci) + irq = chip->pci->irq; + else + irq = platform_get_irq(to_platform_device(chip->card->dev), 0); + + if (request_irq(irq, azx_interrupt, chip->msi ? 0 : IRQF_SHARED, + KBUILD_MODNAME, chip)) { + dev_err(chip->card->dev, + "unable to grab IRQ %d, disabling device\n", irq); + if (do_disconnect) + snd_card_disconnect(chip->card); + return -1; + } + bus->irq = irq; + return 0; +} + +#ifdef CONFIG_PM +static DEFINE_MUTEX(card_list_lock); +static LIST_HEAD(card_list); + +static void azx_add_card_list(struct azx *chip) +{ + struct hda_loongson *hda = container_of(chip, struct hda_loongson, chip); + mutex_lock(&card_list_lock); + list_add(&hda->list, &card_list); + mutex_unlock(&card_list_lock); +} + +static void azx_del_card_list(struct azx *chip) +{ + struct hda_loongson *hda = container_of(chip, struct hda_loongson, chip); + mutex_lock(&card_list_lock); + list_del_init(&hda->list); + mutex_unlock(&card_list_lock); +} + +/* trigger power-save check at writing parameter */ +static int param_set_xint(const char *val, const struct kernel_param *kp) +{ + struct hda_loongson *hda; + struct azx *chip; + int prev = power_save; + int ret = param_set_int(val, kp); + + if (ret || prev == power_save) + return ret; + + mutex_lock(&card_list_lock); + list_for_each_entry(hda, &card_list, list) { + chip = &hda->chip; + if (!hda->probe_continued || chip->disabled) + continue; + snd_hda_set_power_save(&chip->bus, power_save * 1000); + } + mutex_unlock(&card_list_lock); + return 0; +} +#else +#define azx_add_card_list(chip) /* NOP */ +#define azx_del_card_list(chip) /* NOP */ +#endif /* CONFIG_PM */ + +#if defined(CONFIG_PM_SLEEP) +/* + * power management + */ +static int azx_suspend(struct device *dev) +{ + struct snd_card *card = dev_get_drvdata(dev); + struct azx *chip = card->private_data; + struct hdac_bus *bus; + + if (chip->disabled) + return 0; + + bus = azx_bus(chip); + snd_power_change_state(card, SNDRV_CTL_POWER_D3hot); + azx_clear_irq_pending(chip); + azx_stop_chip(chip); + azx_enter_link_reset(chip); + if (bus->irq >= 0) { + free_irq(bus->irq, chip); + bus->irq = -1; + } + return 0; +} + +static int azx_resume(struct device *dev) +{ + struct snd_card *card = dev_get_drvdata(dev); + struct azx *chip = card->private_data; + + if (chip->disabled) + return 0; + + chip->msi = 0; + if (azx_acquire_irq(chip, 1) < 0) + return -EIO; + + azx_init_chip(chip, true); + + snd_power_change_state(card, SNDRV_CTL_POWER_D0); + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM +static int azx_runtime_suspend(struct device *dev) +{ + struct snd_card *card = dev_get_drvdata(dev); + struct azx *chip = card->private_data; + + if (chip->disabled) + return 0; + + if (!azx_has_pm_runtime(chip)) + return 0; + + /* enable controller wake up event */ + azx_writew(chip, WAKEEN, azx_readw(chip, WAKEEN) | + STATESTS_INT_MASK); + + azx_stop_chip(chip); + azx_enter_link_reset(chip); + azx_clear_irq_pending(chip); + return 0; +} + +static int azx_runtime_resume(struct device *dev) +{ + struct snd_card *card = dev_get_drvdata(dev); + struct azx *chip = card->private_data; + struct hda_codec *codec; + int status; + + if (chip->disabled) + return 0; + + if (!azx_has_pm_runtime(chip)) + return 0; + + /* Read STATESTS before controller reset */ + status = azx_readw(chip, STATESTS); + + azx_init_chip(chip, true); + + if (status) { + list_for_each_codec(codec, &chip->bus) + if (status & (1 << codec->addr)) + schedule_delayed_work(&codec->jackpoll_work, + codec->jackpoll_interval); + } + + /* disable controller Wake Up event*/ + azx_writew(chip, WAKEEN, azx_readw(chip, WAKEEN) & + ~STATESTS_INT_MASK); + + return 0; +} + +static int azx_runtime_idle(struct device *dev) +{ + struct snd_card *card = dev_get_drvdata(dev); + struct azx *chip = card->private_data; + + if (chip->disabled) + return 0; + + if (!power_save_controller || !azx_has_pm_runtime(chip) || + azx_bus(chip)->codec_powered) + return -EBUSY; + + return 0; +} + +#endif /* CONFIG_PM */ + +#ifdef CONFIG_PM +static const struct dev_pm_ops azx_pm = { + SET_SYSTEM_SLEEP_PM_OPS(azx_suspend, azx_resume) + SET_RUNTIME_PM_OPS(azx_runtime_suspend, azx_runtime_resume, azx_runtime_idle) +}; + +#define AZX_PM_OPS &azx_pm +#else +#define AZX_PM_OPS NULL +#endif /* CONFIG_PM */ + +static int azx_probe_continue(struct azx *chip); + +/* + * destructor + */ +static int azx_free(struct azx *chip) +{ + struct device *snddev = chip->card->dev; + struct hda_loongson *hda = container_of(chip, struct hda_loongson, chip); + struct hdac_bus *bus = azx_bus(chip); + + if (azx_has_pm_runtime(chip) && chip->running) + pm_runtime_get_noresume(snddev); + + azx_del_card_list(chip); + + complete_all(&hda->probe_wait); + + if (bus->chip_init) { + azx_clear_irq_pending(chip); + azx_stop_all_streams(chip); + azx_stop_chip(chip); + } + + if (bus->irq >= 0) + free_irq(bus->irq, (void*)chip); + if (bus->remap_addr) + iounmap(bus->remap_addr); + + azx_free_stream_pages(chip); + azx_free_streams(chip); + snd_hdac_bus_exit(bus); + kfree(hda); + + return 0; +} + +static int azx_dev_disconnect(struct snd_device *device) +{ + struct azx *chip = device->device_data; + + chip->bus.shutdown = 1; + return 0; +} + +static int azx_dev_free(struct snd_device *device) +{ + return azx_free(device->device_data); +} + +/* + * constructor + */ +static const struct hda_controller_ops loongson_hda_ops; + +static int azx_create(struct snd_card *card, struct pci_dev *pcidev, + struct platform_device *platdev, int dev, + unsigned int driver_caps, struct azx **rchip) +{ + static struct snd_device_ops ops = { + .dev_disconnect = azx_dev_disconnect, + .dev_free = azx_dev_free, + }; + struct hda_loongson *hda; + struct azx *chip; + int err; + + *rchip = NULL; + + hda = kzalloc(sizeof(*hda), GFP_KERNEL); + if (!hda) { + dev_err(card->dev, "Cannot allocate hda\n"); + return -ENOMEM; + } + + chip = &hda->chip; + mutex_init(&chip->open_mutex); + chip->card = card; + chip->pci = pcidev; + chip->ops = &loongson_hda_ops; + chip->driver_caps = driver_caps; + chip->driver_type = driver_caps & 0xff; + chip->dev_index = dev; + if (jackpoll_ms[dev] >= 50 && jackpoll_ms[dev] <= 60000) + chip->jackpoll_interval = msecs_to_jiffies(jackpoll_ms[dev]); + INIT_LIST_HEAD(&chip->pcm_list); + INIT_WORK(&hda->irq_pending_work, azx_irq_pending_work); + INIT_LIST_HEAD(&hda->list); + init_completion(&hda->probe_wait); + + chip->get_position[0] = chip->get_position[1] = azx_get_pos_lpib; + + chip->snoop = false; + chip->single_cmd = single_cmd; + azx_bus(chip)->codec_mask = chip->codec_probe_mask = 0xf; + + if (bdl_pos_adj[dev] < 0) { + switch (chip->driver_type) { + case AZX_DRIVER_LS7A: + bdl_pos_adj[dev] = 1; + break; + default: + bdl_pos_adj[dev] = 32; + break; + } + } + chip->bdl_pos_adj = bdl_pos_adj[dev]; + + err = azx_bus_init(chip, model[dev]); + if (err < 0) { + kfree(hda); + return err; + } + + err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, chip, &ops); + if (err < 0) { + dev_err(card->dev, "Error creating device [card]!\n"); + azx_free(chip); + return err; + } + + *rchip = chip; + + return 0; +} + +static int azx_first_init(struct azx *chip) +{ + int dev = chip->dev_index; + struct snd_card *card = chip->card; + struct hdac_bus *bus = azx_bus(chip); + int err; + unsigned short gcap; + + bus->addr = pci_resource_start(chip->pci, 0); + bus->remap_addr = ioremap(bus->addr, + pci_resource_end(chip->pci, 0) - pci_resource_start(chip->pci, 0) + 1); + if (bus->remap_addr == NULL) { + dev_err(card->dev, "ioremap error\n"); + return -ENXIO; + } + + chip->msi = 0; + + if (azx_acquire_irq(chip, 0) < 0) + return -EBUSY; + + synchronize_irq(bus->irq); + + gcap = azx_readw(chip, GCAP); + dev_dbg(card->dev, "chipset global capabilities = 0x%x\n", gcap); + + /* disable 64bit DMA address on some devices */ + if (chip->driver_caps & AZX_DCAPS_NO_64BIT) { + dev_dbg(card->dev, "Disabling 64bit DMA\n"); + gcap &= ~AZX_GCAP_64OK; + } + + /* disable buffer size rounding to 128-byte multiples if supported */ + if (align_buffer_size >= 0) + chip->align_buffer_size = !!align_buffer_size; + else { + if (chip->driver_caps & AZX_DCAPS_NO_ALIGN_BUFSIZE) + chip->align_buffer_size = 0; + else + chip->align_buffer_size = 1; + } + + /* allow 64bit DMA address if supported by H/W */ + if ((gcap & AZX_GCAP_64OK) && !dma_set_mask(chip->card->dev, DMA_BIT_MASK(64))) + dma_set_coherent_mask(chip->card->dev, DMA_BIT_MASK(64)); + else { + dma_set_mask(chip->card->dev, DMA_BIT_MASK(32)); + dma_set_coherent_mask(chip->card->dev, DMA_BIT_MASK(32)); + } + + /* read number of streams from GCAP register instead of using + * hardcoded value + */ + chip->capture_streams = (gcap >> 8) & 0x0f; + chip->playback_streams = (gcap >> 12) & 0x0f; + if (!chip->playback_streams && !chip->capture_streams) { + /* gcap didn't give any info, switching to old method */ + chip->capture_streams = LS7A_NUM_CAPTURE; + chip->playback_streams = LS7A_NUM_PLAYBACK; + } + chip->capture_index_offset = 0; + chip->playback_index_offset = chip->capture_streams; + chip->num_streams = chip->playback_streams + chip->capture_streams; + + /* initialize streams */ + err = azx_init_streams(chip); + if (err < 0) + return err; + chip->playback_streams = chip->capture_streams = 1; /* Loongson */ + + err = azx_alloc_stream_pages(chip); + if (err < 0) + return err; + + /* initialize chip */ + azx_init_chip(chip, (probe_only[dev] & 2) == 0); + + /* codec detection */ + if (!azx_bus(chip)->codec_mask) { + dev_err(card->dev, "no codecs found!\n"); + return -ENODEV; + } + + strcpy(card->driver, "HDA-Loongson"); + strlcpy(card->shortname, driver_short_names[chip->driver_type], + sizeof(card->shortname)); + snprintf(card->longname, sizeof(card->longname), + "%s at 0x%lx irq %i", + card->shortname, bus->addr, bus->irq); + + return 0; +} + +static const struct hda_controller_ops loongson_hda_ops = { + .position_check = azx_position_check, +}; + +/* number of codec slots for each chipset: 0 = default slots (i.e. 4) */ +static unsigned int azx_max_codecs[AZX_NUM_DRIVERS] = {}; + +static int azx_probe_continue(struct azx *chip) +{ + struct hda_loongson *hda = container_of(chip, struct hda_loongson, chip); + int dev = chip->dev_index; + int err; + struct device *snddev = chip->card->dev; + + hda->probe_continued = 1; + + err = azx_first_init(chip); + if (err < 0) + goto out_free; + +#ifdef CONFIG_SND_HDA_INPUT_BEEP + chip->beep_mode = beep_mode[dev]; +#endif + + /* create codec instances */ + err = azx_probe_codecs(chip, azx_max_codecs[chip->driver_type]); + if (err < 0) + goto out_free; + + if ((probe_only[dev] & 1) == 0) { + err = azx_codec_configure(chip); + if (err < 0) + goto out_free; + } + + err = snd_card_register(chip->card); + if (err < 0) + goto out_free; + + chip->running = 1; + azx_add_card_list(chip); +#ifdef CONFIG_PM + pm_runtime_forbid(snddev); + pm_runtime_set_active(snddev); +#endif + snd_hda_set_power_save(&chip->bus, power_save * 1000); + if (azx_has_pm_runtime(chip)) + pm_runtime_put_noidle(snddev); + +out_free: + complete_all(&hda->probe_wait); + return err; +} + +static const struct pci_device_id azx_ids[] = { + {PCI_DEVICE(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_HDA), + .driver_data = AZX_DRIVER_LS7A | AZX_DCAPS_LS2X_WORKAROUND}, + {PCI_DEVICE(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_HDMI), + .driver_data = AZX_DRIVER_HDMI | AZX_DCAPS_LS2X_WORKAROUND}, + {} +}; + +MODULE_DEVICE_TABLE(pci, azx_ids); + +static int azx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) +{ + int ret; + bool probe_now; + static int dev; + struct snd_card *card; + struct azx *chip; + struct hda_loongson *hda; + + /* Enable device in PCI config */ + ret = pci_enable_device(pdev); + if (ret < 0) { + printk(KERN_ERR "Loongson HDA (%s): Cannot enable PCI device\n", + pci_name(pdev)); + goto out; + } + + /* request the mem regions */ + ret = pci_request_region(pdev, 0, "Loongson HDA"); + if (ret < 0) { + printk( KERN_ERR "Loongson HDA (%s): cannot request region 0.\n", + pci_name(pdev)); + goto out; + } + + if (dev >= SNDRV_CARDS) + return -ENODEV; + if (!enable[dev]) { + dev++; + return -ENOENT; + } + + ret = snd_card_new(&pdev->dev, index[dev], id[dev], THIS_MODULE, + 0, &card); + if (ret < 0) { + dev_err(&pdev->dev, "Error creating card!\n"); + return ret; + } + + ret = azx_create(card, pdev, NULL, dev, pid->driver_data, &chip); + if (ret < 0) + goto out_free; + card->private_data = chip; + hda = container_of(chip, struct hda_loongson, chip); + + dev_set_drvdata(&pdev->dev, card); + + probe_now = !chip->disabled; + + if (probe_now) { + ret = azx_probe_continue(chip); + if (ret < 0) + goto out_free; + } + + dev++; + if (chip->disabled) + complete_all(&hda->probe_wait); + return 0; + +out_free: + snd_card_free(card); +out: + return ret; +} + +static void azx_pci_remove(struct pci_dev *pdev) +{ + snd_card_free(dev_get_drvdata(&pdev->dev)); +} + +static void azx_pci_shutdown(struct pci_dev *pdev) +{ + struct snd_card *card = dev_get_drvdata(&pdev->dev); + struct azx *chip; + + if (!card) + return; + chip = card->private_data; + if (chip && chip->running) + azx_stop_chip(chip); +} + +/* pci_driver definition */ +static struct pci_driver azx_pci_driver = { + .name = "loongson-audio", + .id_table = azx_ids, + .probe = azx_pci_probe, + .remove = azx_pci_remove, + .shutdown = azx_pci_shutdown, + .driver.pm = AZX_PM_OPS, +}; + +static int __init alsa_card_azx_init(void) +{ + int ret; + + ret = pci_register_driver(&azx_pci_driver); + if (ret) + pr_err("hda azx pci driver register\n"); + + return ret; +} + +static void __exit alsa_card_azx_exit(void) +{ + pci_unregister_driver(&azx_pci_driver); +} + +module_init(alsa_card_azx_init) +module_exit(alsa_card_azx_exit) diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 0d1c6c4c1ee62f2a632861b153ba0218f6a413f1..47a07acebdcf44192516310f2496d9769beb1242 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -4311,6 +4311,7 @@ static int patch_gf_hdmi(struct hda_codec *codec) * patch entries */ static const struct hda_device_id snd_hda_id_hdmi[] = { +HDA_CODEC_ENTRY(0x00147a47, "Loongson HDMI", patch_generic_hdmi), HDA_CODEC_ENTRY(0x1002793c, "RS600 HDMI", patch_atihdmi), HDA_CODEC_ENTRY(0x10027919, "RS600 HDMI", patch_atihdmi), HDA_CODEC_ENTRY(0x1002791a, "RS690/780 HDMI", patch_atihdmi), diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index c7e76bfad84d11ec239bf2168202d029fa02b446..185a1252d5d5370d6f5ca39b892dba0a473cf0bc 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -25,6 +25,7 @@ #include "hda_auto_parser.h" #include "hda_jack.h" #include "hda_generic.h" +#include "hda_controller.h" /* keep halting ALC5505 DSP, for power saving */ #define HALT_REALTEK_ALC5505 @@ -344,6 +345,13 @@ static void alc_fixup_micmute_led(struct hda_codec *codec, snd_hda_gen_add_micmute_led_cdev(codec, NULL); } +int has_loongson_workaround(struct hda_codec *codec) +{ + struct azx *chip = bus_to_azx(&codec->bus->core); + + return chip->driver_caps & AZX_DCAPS_LS2X_WORKAROUND; +} + /* * Fix hardware PLL issue * On some codecs, the analog PLL gating control must be off while @@ -685,10 +693,10 @@ static int alc_auto_parse_customize_define(struct hda_codec *codec) goto do_sku; } - if (!codec->bus->pci) + if (!codec->bus->pci && !has_loongson_workaround(codec)) return -1; ass = codec->core.subsystem_id & 0xffff; - if (ass != codec->bus->pci->subsystem_device && (ass & 1)) + if (codec->bus->pci && ass != codec->bus->pci->subsystem_device && (ass & 1)) goto do_sku; nid = 0x1d; @@ -9034,6 +9042,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x87f5, "HP", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87f6, "HP Spectre x360 14", ALC245_FIXUP_HP_X360_AMP), SND_PCI_QUIRK(0x103c, 0x87f7, "HP Spectre x360 14", ALC245_FIXUP_HP_X360_AMP), + SND_PCI_QUIRK(0x103c, 0x87fe, "HP Laptop 15s-fq2xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8805, "HP ProBook 650 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x880d, "HP EliteBook 830 G8 Notebook PC", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8811, "HP Spectre x360 15-eb1xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), diff --git a/sound/pci/oxygen/oxygen_mixer.c b/sound/pci/oxygen/oxygen_mixer.c index 46705ec77b4810ae0fd958749580937db8830ce3..eb3aca16359c58f6f5985ba44ef2173b3d5afc50 100644 --- a/sound/pci/oxygen/oxygen_mixer.c +++ b/sound/pci/oxygen/oxygen_mixer.c @@ -718,7 +718,7 @@ static int ac97_fp_rec_volume_put(struct snd_kcontrol *ctl, oldreg = oxygen_read_ac97(chip, 1, AC97_REC_GAIN); newreg = oldreg & ~0x0707; newreg = newreg | (value->value.integer.value[0] & 7); - newreg = newreg | ((value->value.integer.value[0] & 7) << 8); + newreg = newreg | ((value->value.integer.value[1] & 7) << 8); change = newreg != oldreg; if (change) oxygen_write_ac97(chip, 1, AC97_REC_GAIN, newreg); diff --git a/sound/soc/atmel/sam9g20_wm8731.c b/sound/soc/atmel/sam9g20_wm8731.c index d243de5f23dc1c4fae8ad88566072ab28b88e803..8a55d59a6c2aa97bc7680287479d0a1c9dd2859d 100644 --- a/sound/soc/atmel/sam9g20_wm8731.c +++ b/sound/soc/atmel/sam9g20_wm8731.c @@ -46,6 +46,35 @@ */ #undef ENABLE_MIC_INPUT +static struct clk *mclk; + +static int at91sam9g20ek_set_bias_level(struct snd_soc_card *card, + struct snd_soc_dapm_context *dapm, + enum snd_soc_bias_level level) +{ + static int mclk_on; + int ret = 0; + + switch (level) { + case SND_SOC_BIAS_ON: + case SND_SOC_BIAS_PREPARE: + if (!mclk_on) + ret = clk_enable(mclk); + if (ret == 0) + mclk_on = 1; + break; + + case SND_SOC_BIAS_OFF: + case SND_SOC_BIAS_STANDBY: + if (mclk_on) + clk_disable(mclk); + mclk_on = 0; + break; + } + + return ret; +} + static const struct snd_soc_dapm_widget at91sam9g20ek_dapm_widgets[] = { SND_SOC_DAPM_MIC("Int Mic", NULL), SND_SOC_DAPM_SPK("Ext Spk", NULL), @@ -106,6 +135,7 @@ static struct snd_soc_card snd_soc_at91sam9g20ek = { .owner = THIS_MODULE, .dai_link = &at91sam9g20ek_dai, .num_links = 1, + .set_bias_level = at91sam9g20ek_set_bias_level, .dapm_widgets = at91sam9g20ek_dapm_widgets, .num_dapm_widgets = ARRAY_SIZE(at91sam9g20ek_dapm_widgets), @@ -118,6 +148,7 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct device_node *codec_np, *cpu_np; + struct clk *pllb; struct snd_soc_card *card = &snd_soc_at91sam9g20ek; int ret; @@ -131,6 +162,31 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev) return -EINVAL; } + /* + * Codec MCLK is supplied by PCK0 - set it up. + */ + mclk = clk_get(NULL, "pck0"); + if (IS_ERR(mclk)) { + dev_err(&pdev->dev, "Failed to get MCLK\n"); + ret = PTR_ERR(mclk); + goto err; + } + + pllb = clk_get(NULL, "pllb"); + if (IS_ERR(pllb)) { + dev_err(&pdev->dev, "Failed to get PLLB\n"); + ret = PTR_ERR(pllb); + goto err_mclk; + } + ret = clk_set_parent(mclk, pllb); + clk_put(pllb); + if (ret != 0) { + dev_err(&pdev->dev, "Failed to set MCLK parent\n"); + goto err_mclk; + } + + clk_set_rate(mclk, MCLK_RATE); + card->dev = &pdev->dev; /* Parse device node info */ @@ -174,6 +230,9 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev) return ret; +err_mclk: + clk_put(mclk); + mclk = NULL; err: atmel_ssc_put_audio(0); return ret; @@ -183,6 +242,8 @@ static int at91sam9g20ek_audio_remove(struct platform_device *pdev) { struct snd_soc_card *card = platform_get_drvdata(pdev); + clk_disable(mclk); + mclk = NULL; snd_soc_unregister_card(card); atmel_ssc_put_audio(0); diff --git a/tools/arch/loongarch/include/uapi/asm/bitsperlong.h b/tools/arch/loongarch/include/uapi/asm/bitsperlong.h new file mode 100644 index 0000000000000000000000000000000000000000..00b4ba1e5cdf032f81aff9f728dcffea68571880 --- /dev/null +++ b/tools/arch/loongarch/include/uapi/asm/bitsperlong.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_LOONGARCH_BITSPERLONG_H +#define __ASM_LOONGARCH_BITSPERLONG_H + +#define __BITS_PER_LONG (__SIZEOF_LONG__ * 8) + +#include + +#endif /* __ASM_LOONGARCH_BITSPERLONG_H */ diff --git a/tools/arch/loongarch/include/uapi/asm/perf_regs.h b/tools/arch/loongarch/include/uapi/asm/perf_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..29d69c00fc7a665f743d931223916c43cd0b4849 --- /dev/null +++ b/tools/arch/loongarch/include/uapi/asm/perf_regs.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASM_LOONGARCH_PERF_REGS_H +#define _ASM_LOONGARCH_PERF_REGS_H + +enum perf_event_loongarch_regs { + PERF_REG_LOONGARCH_PC, + PERF_REG_LOONGARCH_R1, + PERF_REG_LOONGARCH_R2, + PERF_REG_LOONGARCH_R3, + PERF_REG_LOONGARCH_R4, + PERF_REG_LOONGARCH_R5, + PERF_REG_LOONGARCH_R6, + PERF_REG_LOONGARCH_R7, + PERF_REG_LOONGARCH_R8, + PERF_REG_LOONGARCH_R9, + PERF_REG_LOONGARCH_R10, + PERF_REG_LOONGARCH_R11, + PERF_REG_LOONGARCH_R12, + PERF_REG_LOONGARCH_R13, + PERF_REG_LOONGARCH_R14, + PERF_REG_LOONGARCH_R15, + PERF_REG_LOONGARCH_R16, + PERF_REG_LOONGARCH_R17, + PERF_REG_LOONGARCH_R18, + PERF_REG_LOONGARCH_R19, + PERF_REG_LOONGARCH_R20, + PERF_REG_LOONGARCH_R21, + PERF_REG_LOONGARCH_R22, + PERF_REG_LOONGARCH_R23, + PERF_REG_LOONGARCH_R24, + PERF_REG_LOONGARCH_R25, + PERF_REG_LOONGARCH_R26, + PERF_REG_LOONGARCH_R27, + PERF_REG_LOONGARCH_R28, + PERF_REG_LOONGARCH_R29, + PERF_REG_LOONGARCH_R30, + PERF_REG_LOONGARCH_R31, + PERF_REG_LOONGARCH_MAX, +}; +#endif /* _ASM_LOONGARCH_PERF_REGS_H */ diff --git a/tools/arch/loongarch/include/uapi/asm/unistd.h b/tools/arch/loongarch/include/uapi/asm/unistd.h new file mode 100644 index 0000000000000000000000000000000000000000..d3666a55f7a66df540671e42f272c261d353b677 --- /dev/null +++ b/tools/arch/loongarch/include/uapi/asm/unistd.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (C) 2020-2021 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_SYS_CLONE +#define __ARCH_WANT_SYS_CLONE3 + +#include diff --git a/tools/include/uapi/asm/bitsperlong.h b/tools/include/uapi/asm/bitsperlong.h index edba4d93e9e6a346572d12e7def7e01f1c88822d..da5206517158109f5b3db67f76fae1ee71b80b27 100644 --- a/tools/include/uapi/asm/bitsperlong.h +++ b/tools/include/uapi/asm/bitsperlong.h @@ -17,6 +17,8 @@ #include "../../../arch/riscv/include/uapi/asm/bitsperlong.h" #elif defined(__alpha__) #include "../../../arch/alpha/include/uapi/asm/bitsperlong.h" +#elif defined(__loongarch__) +#include "../../../arch/loongarch/include/uapi/asm/bitsperlong.h" #else #include #endif diff --git a/tools/lib/api/io.h b/tools/lib/api/io.h index 777c20f6b6047b68593f00ec074db54f50e85bfe..458acd294237d1acc3c515b038aad370789a9499 100644 --- a/tools/lib/api/io.h +++ b/tools/lib/api/io.h @@ -9,6 +9,7 @@ #include #include +#include struct io { /* File descriptor being read/ */ diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index f05cfc082915dcec308e3d17f309406f34399b8c..4303b31498d81852cc22c440c9e7b289005a60d8 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -101,6 +101,7 @@ enum bpf_enum_value_kind { case 2: val = *(const unsigned short *)p; break; \ case 4: val = *(const unsigned int *)p; break; \ case 8: val = *(const unsigned long long *)p; break; \ + default: val = 0; break; \ } \ val <<= __CORE_RELO(s, field, LSHIFT_U64); \ if (__CORE_RELO(s, field, SIGNED)) \ diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 89905b4e93091e82aca7947371eefd844df12445..2772085f059f5c245b162e4752dcc27e95c11191 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -32,7 +32,7 @@ ifneq ($(NO_SYSCALL_TABLE),1) NO_SYSCALL_TABLE := 0 endif else - ifeq ($(SRCARCH),$(filter $(SRCARCH),powerpc arm64 s390)) + ifeq ($(SRCARCH),$(filter $(SRCARCH),powerpc arm64 s390 loongarch)) NO_SYSCALL_TABLE := 0 endif endif @@ -74,6 +74,12 @@ ifeq ($(SRCARCH),arm64) LIBUNWIND_LIBS = -lunwind -lunwind-aarch64 endif +ifeq ($(SRCARCH),loongarch) + NO_PERF_REGS := 0 + CFLAGS += -I$(OUTPUT)arch/loongarch/include/generated + LIBUNWIND_LIBS = -lunwind -lunwind-loongarch64 +endif + ifeq ($(SRCARCH),riscv) NO_PERF_REGS := 0 endif @@ -95,7 +101,7 @@ endif # Disable it on all other architectures in case libdw unwind # support is detected in system. Add supported architectures # to the check. -ifneq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc s390 csky riscv)) +ifneq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc s390 csky riscv loongarch)) NO_LIBDW_DWARF_UNWIND := 1 endif @@ -117,7 +123,7 @@ endef ifdef LIBUNWIND_DIR LIBUNWIND_CFLAGS = -I$(LIBUNWIND_DIR)/include LIBUNWIND_LDFLAGS = -L$(LIBUNWIND_DIR)/lib - LIBUNWIND_ARCHS = x86 x86_64 arm aarch64 debug-frame-arm debug-frame-aarch64 + LIBUNWIND_ARCHS = x86 x86_64 arm aarch64 debug-frame-arm debug-frame-aarch64 loongarch $(foreach libunwind_arch,$(LIBUNWIND_ARCHS),$(call libunwind_arch_set_flags,$(libunwind_arch))) endif diff --git a/tools/perf/arch/loongarch/Build b/tools/perf/arch/loongarch/Build new file mode 100644 index 0000000000000000000000000000000000000000..e4e5f33c84d862aaba8e05ded73e0c336d6e90bd --- /dev/null +++ b/tools/perf/arch/loongarch/Build @@ -0,0 +1 @@ +perf-y += util/ diff --git a/tools/perf/arch/loongarch/Makefile b/tools/perf/arch/loongarch/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c392e7af474332e2c6ebd5a2e108207fc08e4786 --- /dev/null +++ b/tools/perf/arch/loongarch/Makefile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +ifndef NO_DWARF +PERF_HAVE_DWARF_REGS := 1 +endif +PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 +PERF_HAVE_JITDUMP := 1 + +# +# Syscall table generation for perf +# + +out := $(OUTPUT)arch/loongarch/include/generated/asm +header := $(out)/syscalls.c +incpath := $(srctree)/tools +sysdef := $(srctree)/tools/arch/loongarch/include/uapi/asm/unistd.h +sysprf := $(srctree)/tools/perf/arch/loongarch/entry/syscalls/ +systbl := $(sysprf)/mksyscalltbl + +# Create output directory if not already present +_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') + +$(header): $(sysdef) $(systbl) + $(Q)$(SHELL) '$(systbl)' '$(CC)' '$(HOSTCC)' $(incpath) $(sysdef) > $@ + +clean:: + $(call QUIET_CLEAN, loongarch) $(RM) $(header) + +archheaders: $(header) diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c new file mode 100644 index 0000000000000000000000000000000000000000..206e1fca38a47350cde878f8f40f50039d50e0b6 --- /dev/null +++ b/tools/perf/arch/loongarch/annotate/instructions.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Perf annotate functions. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + */ + +static +struct ins_ops *loongarch__associate_ins_ops(struct arch *arch, const char *name) +{ + struct ins_ops *ops = NULL; + + if (!strncmp(name, "beqz", 4) || + !strncmp(name, "bnez", 4) || + !strncmp(name, "beq", 3) || + !strncmp(name, "bne", 3) || + !strncmp(name, "blt", 3) || + !strncmp(name, "bge", 3) || + !strncmp(name, "bltu", 4) || + !strncmp(name, "bgeu", 4) || + !strncmp(name, "bl", 2)) + ops = &call_ops; + else if (!strncmp(name, "jirl", 4)) + ops = &ret_ops; + else if (name[0] == 'b') + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +static +int loongarch__annotate_init(struct arch *arch, char *cpuid __maybe_unused) +{ + if (!arch->initialized) { + arch->associate_instruction_ops = loongarch__associate_ins_ops; + arch->initialized = true; + arch->objdump.comment_char = '#'; + } + + return 0; +} diff --git a/tools/perf/arch/loongarch/entry/syscalls/mksyscalltbl b/tools/perf/arch/loongarch/entry/syscalls/mksyscalltbl new file mode 100755 index 0000000000000000000000000000000000000000..86dad5a018b73335855428b400656e73cbbc55bf --- /dev/null +++ b/tools/perf/arch/loongarch/entry/syscalls/mksyscalltbl @@ -0,0 +1,60 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Generate system call table for perf. Derived from +# powerpc script. +# +# Copyright (C) 2020 Loongson Technology Co., Ltd. +# Author(s): Ming Wang + +gcc=$1 +hostcc=$2 +incpath=$3 +input=$4 + +if ! test -r $input; then + echo "Could not read input file" >&2 + exit 1 +fi + +create_table_from_c() +{ + local sc nr last_sc + + create_table_exe=`mktemp ${TMPDIR:-/tmp}/create-table-XXXXXX` + + { + + cat <<-_EoHEADER + #include + #include "$input" + int main(int argc, char *argv[]) + { + _EoHEADER + + while read sc nr; do + printf "%s\n" " printf(\"\\t[%d] = \\\"$sc\\\",\\n\", $nr);" + last_sc=$nr + done + + printf "%s\n" " printf(\"#define SYSCALLTBL_LOONGARCH_MAX_ID %d\\n\", $last_sc);" + printf "}\n" + + } | $hostcc -I $incpath/include/uapi -o $create_table_exe -x c - + + $create_table_exe + + rm -f $create_table_exe +} + +create_table() +{ + echo "static const char *syscalltbl_loongarch[] = {" + create_table_from_c + echo "};" +} + +$gcc -E -dM -x c -I $incpath/include/uapi $input \ + |sed -ne 's/^#define __NR_//p' \ + |sort -t' ' -k2 -n \ + |create_table diff --git a/tools/perf/arch/loongarch/include/dwarf-regs-table.h b/tools/perf/arch/loongarch/include/dwarf-regs-table.h new file mode 100644 index 0000000000000000000000000000000000000000..676c54a226a5d56854ea8ed029e87a5a02315d55 --- /dev/null +++ b/tools/perf/arch/loongarch/include/dwarf-regs-table.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * dwarf-regs-table.h : Mapping of DWARF debug register numbers into + * register names. + * + * Copyright (C) 2020 Loongson Technology Corporation Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifdef DEFINE_DWARF_REGSTR_TABLE +static const char * const loongarch_regstr_tbl[] = { + "$0", "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$9", + "$10", "$11", "$12", "$13", "$14", "$15", "$16", "$17", "$18", "$19", + "$20", "$21", "$22", "$23", "$24", "$25", "$26", "$27", "$28", "%29", + "$30", "$31", +}; +#endif diff --git a/tools/perf/arch/loongarch/include/perf_regs.h b/tools/perf/arch/loongarch/include/perf_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..82d531dcd90fb82b5d63d468f0d36eda88afa6c3 --- /dev/null +++ b/tools/perf/arch/loongarch/include/perf_regs.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_PERF_REGS_H +#define ARCH_PERF_REGS_H + +#include +#include +#include + +#define PERF_REGS_MAX PERF_REG_LOONGARCH_MAX +#define PERF_REG_IP PERF_REG_LOONGARCH_PC +#define PERF_REG_SP PERF_REG_LOONGARCH_R3 + +#define PERF_REGS_MASK ((1ULL << PERF_REG_LOONGARCH_MAX) - 1) + +static inline const char *__perf_reg_name(int id) +{ + switch (id) { + case PERF_REG_LOONGARCH_PC: + return "PC"; + case PERF_REG_LOONGARCH_R1: + return "$1"; + case PERF_REG_LOONGARCH_R2: + return "$2"; + case PERF_REG_LOONGARCH_R3: + return "$3"; + case PERF_REG_LOONGARCH_R4: + return "$4"; + case PERF_REG_LOONGARCH_R5: + return "$5"; + case PERF_REG_LOONGARCH_R6: + return "$6"; + case PERF_REG_LOONGARCH_R7: + return "$7"; + case PERF_REG_LOONGARCH_R8: + return "$8"; + case PERF_REG_LOONGARCH_R9: + return "$9"; + case PERF_REG_LOONGARCH_R10: + return "$10"; + case PERF_REG_LOONGARCH_R11: + return "$11"; + case PERF_REG_LOONGARCH_R12: + return "$12"; + case PERF_REG_LOONGARCH_R13: + return "$13"; + case PERF_REG_LOONGARCH_R14: + return "$14"; + case PERF_REG_LOONGARCH_R15: + return "$15"; + case PERF_REG_LOONGARCH_R16: + return "$16"; + case PERF_REG_LOONGARCH_R17: + return "$17"; + case PERF_REG_LOONGARCH_R18: + return "$18"; + case PERF_REG_LOONGARCH_R19: + return "$19"; + case PERF_REG_LOONGARCH_R20: + return "$20"; + case PERF_REG_LOONGARCH_R21: + return "$21"; + case PERF_REG_LOONGARCH_R22: + return "$22"; + case PERF_REG_LOONGARCH_R23: + return "$23"; + case PERF_REG_LOONGARCH_R24: + return "$24"; + case PERF_REG_LOONGARCH_R25: + return "$25"; + case PERF_REG_LOONGARCH_R26: + return "$26"; + case PERF_REG_LOONGARCH_R27: + return "$27"; + case PERF_REG_LOONGARCH_R28: + return "$28"; + case PERF_REG_LOONGARCH_R29: + return "$29"; + case PERF_REG_LOONGARCH_R30: + return "$30"; + case PERF_REG_LOONGARCH_R31: + return "$31"; + default: + break; + } + return NULL; +} + +#endif /* ARCH_PERF_REGS_H */ diff --git a/tools/perf/arch/loongarch/util/Build b/tools/perf/arch/loongarch/util/Build new file mode 100644 index 0000000000000000000000000000000000000000..d776125a2d06832b5841c798bf39460357737ce9 --- /dev/null +++ b/tools/perf/arch/loongarch/util/Build @@ -0,0 +1,5 @@ +perf-y += perf_regs.o + +perf-$(CONFIG_DWARF) += dwarf-regs.o +perf-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o +perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/arch/loongarch/util/dwarf-regs.c b/tools/perf/arch/loongarch/util/dwarf-regs.c new file mode 100644 index 0000000000000000000000000000000000000000..0509cfd597d64f335af833439cbb892eb26851a9 --- /dev/null +++ b/tools/perf/arch/loongarch/util/dwarf-regs.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * + * Copyright (C) 2013 Cavium, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include /* for EINVAL */ +#include /* for strcmp */ +#include + +struct pt_regs_dwarfnum { + const char *name; + unsigned int dwarfnum; +}; + +static struct pt_regs_dwarfnum loongarch_gpr_table[] = { + {"$0", 0}, {"$1", 1}, {"$2", 2}, {"$3", 3}, + {"$4", 4}, {"$5", 5}, {"$6", 6}, {"$7", 7}, + {"$8", 8}, {"$9", 9}, {"$10", 10}, {"$11", 11}, + {"$12", 12}, {"$13", 13}, {"$14", 14}, {"$15", 15}, + {"$16", 16}, {"$17", 17}, {"$18", 18}, {"$19", 19}, + {"$20", 20}, {"$21", 21}, {"$22", 22}, {"$23", 23}, + {"$24", 24}, {"$25", 25}, {"$26", 26}, {"$27", 27}, + {"$28", 28}, {"$29", 29}, {"$30", 30}, {"$31", 31}, + {NULL, 0} +}; + +const char *get_arch_regstr(unsigned int n) +{ + n %= 32; + return loongarch_gpr_table[n].name; +} + +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_dwarfnum *roff; + + for (roff = loongarch_gpr_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->dwarfnum; + return -EINVAL; +} diff --git a/tools/perf/arch/loongarch/util/perf_regs.c b/tools/perf/arch/loongarch/util/perf_regs.c new file mode 100644 index 0000000000000000000000000000000000000000..2833e101a7c6407263130e9948a06a2caa32bc4b --- /dev/null +++ b/tools/perf/arch/loongarch/util/perf_regs.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../../../util/perf_regs.h" + +const struct sample_reg sample_reg_masks[] = { + SMPL_REG_END +}; diff --git a/tools/perf/arch/loongarch/util/unwind-libdw.c b/tools/perf/arch/loongarch/util/unwind-libdw.c new file mode 100644 index 0000000000000000000000000000000000000000..a9415385230a12254f70ee213018127df7c37100 --- /dev/null +++ b/tools/perf/arch/loongarch/util/unwind-libdw.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2020-2023 Loongson Technology Corporation Limited */ + +#include +#include "../../util/unwind-libdw.h" +#include "../../util/perf_regs.h" +#include "../../util/sample.h" + +bool libdw__arch_set_initial_registers(Dwfl_Thread *thread, void *arg) +{ + struct unwind_info *ui = arg; + struct regs_dump *user_regs = &ui->sample->user_regs; + Dwarf_Word dwarf_regs[PERF_REG_LOONGARCH_MAX]; + +#define REG(r) ({ \ + Dwarf_Word val = 0; \ + perf_reg_value(&val, user_regs, PERF_REG_LOONGARCH_##r); \ + val; \ +}) + + dwarf_regs[0] = 0; + dwarf_regs[1] = REG(R1); + dwarf_regs[2] = REG(R2); + dwarf_regs[3] = REG(R3); + dwarf_regs[4] = REG(R4); + dwarf_regs[5] = REG(R5); + dwarf_regs[6] = REG(R6); + dwarf_regs[7] = REG(R7); + dwarf_regs[8] = REG(R8); + dwarf_regs[9] = REG(R9); + dwarf_regs[10] = REG(R10); + dwarf_regs[11] = REG(R11); + dwarf_regs[12] = REG(R12); + dwarf_regs[13] = REG(R13); + dwarf_regs[14] = REG(R14); + dwarf_regs[15] = REG(R15); + dwarf_regs[16] = REG(R16); + dwarf_regs[17] = REG(R17); + dwarf_regs[18] = REG(R18); + dwarf_regs[19] = REG(R19); + dwarf_regs[20] = REG(R20); + dwarf_regs[21] = REG(R21); + dwarf_regs[22] = REG(R22); + dwarf_regs[23] = REG(R23); + dwarf_regs[24] = REG(R24); + dwarf_regs[25] = REG(R25); + dwarf_regs[26] = REG(R26); + dwarf_regs[27] = REG(R27); + dwarf_regs[28] = REG(R28); + dwarf_regs[29] = REG(R29); + dwarf_regs[30] = REG(R30); + dwarf_regs[31] = REG(R31); + dwfl_thread_state_register_pc(thread, REG(PC)); + + return dwfl_thread_state_registers(thread, 0, PERF_REG_LOONGARCH_MAX, dwarf_regs); +} diff --git a/tools/perf/arch/loongarch/util/unwind-libunwind.c b/tools/perf/arch/loongarch/util/unwind-libunwind.c new file mode 100644 index 0000000000000000000000000000000000000000..f693167b86efddf976e1448917c49fb3a62a4fdc --- /dev/null +++ b/tools/perf/arch/loongarch/util/unwind-libunwind.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "perf_regs.h" +#include "../../util/unwind.h" +#include "util/debug.h" + +int libunwind__arch_reg_id(int regnum) +{ + switch (regnum) { + case UNW_LOONGARCH64_R1: + return PERF_REG_LOONGARCH_R1; + case UNW_LOONGARCH64_R2: + return PERF_REG_LOONGARCH_R2; + case UNW_LOONGARCH64_R3: + return PERF_REG_LOONGARCH_R3; + case UNW_LOONGARCH64_R4: + return PERF_REG_LOONGARCH_R4; + case UNW_LOONGARCH64_R5: + return PERF_REG_LOONGARCH_R5; + case UNW_LOONGARCH64_R6: + return PERF_REG_LOONGARCH_R6; + case UNW_LOONGARCH64_R7: + return PERF_REG_LOONGARCH_R7; + case UNW_LOONGARCH64_R8: + return PERF_REG_LOONGARCH_R8; + case UNW_LOONGARCH64_R9: + return PERF_REG_LOONGARCH_R9; + case UNW_LOONGARCH64_R10: + return PERF_REG_LOONGARCH_R10; + case UNW_LOONGARCH64_R11: + return PERF_REG_LOONGARCH_R11; + case UNW_LOONGARCH64_R12: + return PERF_REG_LOONGARCH_R12; + case UNW_LOONGARCH64_R13: + return PERF_REG_LOONGARCH_R13; + case UNW_LOONGARCH64_R14: + return PERF_REG_LOONGARCH_R14; + case UNW_LOONGARCH64_R15: + return PERF_REG_LOONGARCH_R15; + case UNW_LOONGARCH64_R16: + return PERF_REG_LOONGARCH_R16; + case UNW_LOONGARCH64_R17: + return PERF_REG_LOONGARCH_R17; + case UNW_LOONGARCH64_R18: + return PERF_REG_LOONGARCH_R18; + case UNW_LOONGARCH64_R19: + return PERF_REG_LOONGARCH_R19; + case UNW_LOONGARCH64_R20: + return PERF_REG_LOONGARCH_R20; + case UNW_LOONGARCH64_R21: + return PERF_REG_LOONGARCH_R21; + case UNW_LOONGARCH64_R22: + return PERF_REG_LOONGARCH_R22; + case UNW_LOONGARCH64_R23: + return PERF_REG_LOONGARCH_R23; + case UNW_LOONGARCH64_R24: + return PERF_REG_LOONGARCH_R24; + case UNW_LOONGARCH64_R25: + return PERF_REG_LOONGARCH_R25; + case UNW_LOONGARCH64_R26: + return PERF_REG_LOONGARCH_R26; + case UNW_LOONGARCH64_R27: + return PERF_REG_LOONGARCH_R27; + case UNW_LOONGARCH64_R28: + return PERF_REG_LOONGARCH_R28; + case UNW_LOONGARCH64_R29: + return PERF_REG_LOONGARCH_R29; + case UNW_LOONGARCH64_R30: + return PERF_REG_LOONGARCH_R30; + case UNW_LOONGARCH64_R31: + return PERF_REG_LOONGARCH_R31; + case UNW_LOONGARCH64_PC: + return PERF_REG_LOONGARCH_PC; + default: + pr_err("unwind: invalid reg id %d\n", regnum); + return -EINVAL; + } + + return -EINVAL; +} diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index 9f085aa09c97aa09508bfa1c2b8783b91b43c7fc..95d3d99e84ac01b5e1ad8cce7b432f14dbef1d7d 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -39,6 +39,7 @@ arch/x86/lib/x86-opcode-map.txt arch/x86/tools/gen-insn-attr-x86.awk arch/arm/include/uapi/asm/perf_regs.h arch/arm64/include/uapi/asm/perf_regs.h +arch/loongarch/include/uapi/asm/perf_regs.h arch/powerpc/include/uapi/asm/perf_regs.h arch/s390/include/uapi/asm/perf_regs.h arch/x86/include/uapi/asm/perf_regs.h diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 1f56a29802e46276a9b71bc663d1fe3395953737..93b56bc42c73de6eece8a6aac95523bd94a7fa4b 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -156,6 +156,7 @@ static int arch__associate_ins_ops(struct arch* arch, const char *name, struct i #include "arch/powerpc/annotate/instructions.c" #include "arch/s390/annotate/instructions.c" #include "arch/sparc/annotate/instructions.c" +#include "arch/loongarch/annotate/instructions.c" static struct arch architectures[] = { { @@ -202,6 +203,13 @@ static struct arch architectures[] = { .comment_char = '#', }, }, + { + .name = "loongarch", + .init = loongarch__annotate_init, + .objdump = { + .comment_char = '#', + }, + }, }; static void ins__delete(struct ins_operands *ops) diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index 4eb02762104bad04908b80119535aed028cb2f09..c50d2c7a264fe3083cd4eec4015900e0f3839bd5 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -533,9 +533,9 @@ int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) return perf_evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); } -void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, - struct perf_env *env, - FILE *fp) +void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, + struct perf_env *env, + FILE *fp) { __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); @@ -551,7 +551,7 @@ void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, if (info->btf_id) { struct btf_node *node; - node = perf_env__find_btf(env, info->btf_id); + node = __perf_env__find_btf(env, info->btf_id); if (node) btf = btf__new((__u8 *)(node->data), node->data_size); diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h index 68f315c3df5bed03c85b7d4e758fe0fd66d7e3bc..50f7412464dfc25f71186568797cfdbd4a2f99a6 100644 --- a/tools/perf/util/bpf-event.h +++ b/tools/perf/util/bpf-event.h @@ -34,9 +34,9 @@ struct btf_node { int machine__process_bpf(struct machine *machine, union perf_event *event, struct perf_sample *sample); int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); -void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, - struct perf_env *env, - FILE *fp); +void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, + struct perf_env *env, + FILE *fp); #else static inline int machine__process_bpf(struct machine *machine __maybe_unused, union perf_event *event __maybe_unused, @@ -51,9 +51,9 @@ static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, return 0; } -static inline void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info __maybe_unused, - struct perf_env *env __maybe_unused, - FILE *fp __maybe_unused) +static inline void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info __maybe_unused, + struct perf_env *env __maybe_unused, + FILE *fp __maybe_unused) { } diff --git a/tools/perf/util/dwarf-regs.c b/tools/perf/util/dwarf-regs.c index 1b49ecee5affd2b19d10ce6790eaa7190e107bc1..43a77aeeb3100730d03504cb8afbe865c33ec751 100644 --- a/tools/perf/util/dwarf-regs.c +++ b/tools/perf/util/dwarf-regs.c @@ -14,6 +14,10 @@ #define EM_AARCH64 183 /* ARM 64 bit */ #endif +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 /* LoongArch */ +#endif + /* Define const char * {arch}_register_tbl[] */ #define DEFINE_DWARF_REGSTR_TABLE #include "../arch/x86/include/dwarf-regs-table.h" @@ -24,6 +28,7 @@ #include "../arch/s390/include/dwarf-regs-table.h" #include "../arch/sparc/include/dwarf-regs-table.h" #include "../arch/xtensa/include/dwarf-regs-table.h" +#include "../arch/loongarch/include/dwarf-regs-table.h" #define __get_dwarf_regstr(tbl, n) (((n) < ARRAY_SIZE(tbl)) ? (tbl)[(n)] : NULL) @@ -53,6 +58,8 @@ const char *get_dwarf_regstr(unsigned int n, unsigned int machine) return __get_dwarf_regstr(sparc_regstr_tbl, n); case EM_XTENSA: return __get_dwarf_regstr(xtensa_regstr_tbl, n); + case EM_LOONGARCH: + return __get_dwarf_regstr(loongarch_regstr_tbl, n); default: pr_err("ELF MACHINE %x is not supported.\n", machine); } diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index 342a83025a863954974608660b9b384759841c72..f115eed2c902cd103da2ef72bc95817b5f058797 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -17,13 +17,19 @@ struct perf_env perf_env; void perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node) +{ + down_write(&env->bpf_progs.lock); + __perf_env__insert_bpf_prog_info(env, info_node); + up_write(&env->bpf_progs.lock); +} + +void __perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node) { __u32 prog_id = info_node->info_linear->info.id; struct bpf_prog_info_node *node; struct rb_node *parent = NULL; struct rb_node **p; - down_write(&env->bpf_progs.lock); p = &env->bpf_progs.infos.rb_node; while (*p != NULL) { @@ -35,15 +41,13 @@ void perf_env__insert_bpf_prog_info(struct perf_env *env, p = &(*p)->rb_right; } else { pr_debug("duplicated bpf prog info %u\n", prog_id); - goto out; + return; } } rb_link_node(&info_node->rb_node, parent, p); rb_insert_color(&info_node->rb_node, &env->bpf_progs.infos); env->bpf_progs.infos_cnt++; -out: - up_write(&env->bpf_progs.lock); } struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, @@ -72,14 +76,22 @@ struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, } bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) +{ + bool ret; + + down_write(&env->bpf_progs.lock); + ret = __perf_env__insert_btf(env, btf_node); + up_write(&env->bpf_progs.lock); + return ret; +} + +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) { struct rb_node *parent = NULL; __u32 btf_id = btf_node->id; struct btf_node *node; struct rb_node **p; - bool ret = true; - down_write(&env->bpf_progs.lock); p = &env->bpf_progs.btfs.rb_node; while (*p != NULL) { @@ -91,25 +103,31 @@ bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) p = &(*p)->rb_right; } else { pr_debug("duplicated btf %u\n", btf_id); - ret = false; - goto out; + return false; } } rb_link_node(&btf_node->rb_node, parent, p); rb_insert_color(&btf_node->rb_node, &env->bpf_progs.btfs); env->bpf_progs.btfs_cnt++; -out: - up_write(&env->bpf_progs.lock); - return ret; + return true; } struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id) +{ + struct btf_node *res; + + down_read(&env->bpf_progs.lock); + res = __perf_env__find_btf(env, btf_id); + up_read(&env->bpf_progs.lock); + return res; +} + +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id) { struct btf_node *node = NULL; struct rb_node *n; - down_read(&env->bpf_progs.lock); n = env->bpf_progs.btfs.rb_node; while (n) { @@ -119,13 +137,9 @@ struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id) else if (btf_id > node->id) n = n->rb_right; else - goto out; + return node; } - node = NULL; - -out: - up_read(&env->bpf_progs.lock); - return node; + return NULL; } /* purge data in bpf_progs.infos tree */ @@ -345,6 +359,8 @@ static const char *normalize_arch(char *arch) return "mips"; if (!strncmp(arch, "sh", 2) && isdigit(arch[2])) return "sh"; + if (!strncmp(arch, "loongarch", 9)) + return "loongarch"; return arch; } diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index 01378a955dd5e8652eb520ecb4a015e496d9b481..ef0fd544cd672c310a953a6086c1ebdb0363618e 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -139,12 +139,16 @@ const char *perf_env__raw_arch(struct perf_env *env); int perf_env__nr_cpus_avail(struct perf_env *env); void perf_env__init(struct perf_env *env); +void __perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node); void perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node); struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, __u32 prog_id); bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id); +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id); int perf_env__numa_node(struct perf_env *env, int cpu); #endif /* __PERF_ENV_H */ diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index 02cd9f75e3d2f8c913a9bbea57cb34f44b8002c4..89a85601485d909462ecf572567cc7f2d8fb8132 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -291,9 +291,9 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, */ phdr = elf_newphdr(e, 1); phdr[0].p_type = PT_LOAD; - phdr[0].p_offset = 0; - phdr[0].p_vaddr = 0; - phdr[0].p_paddr = 0; + phdr[0].p_offset = GEN_ELF_TEXT_OFFSET; + phdr[0].p_vaddr = GEN_ELF_TEXT_OFFSET; + phdr[0].p_paddr = GEN_ELF_TEXT_OFFSET; phdr[0].p_filesz = csize; phdr[0].p_memsz = csize; phdr[0].p_flags = PF_X | PF_R; diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h index ac638945b4cb095e85659b76cf40b40687d6e60e..43dac6998d2b5d02dae4a40de3a89190ca660101 100644 --- a/tools/perf/util/genelf.h +++ b/tools/perf/util/genelf.h @@ -38,6 +38,9 @@ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_ent #elif defined(__s390x__) #define GEN_ELF_ARCH EM_S390 #define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__loongarch__) +#define GEN_ELF_ARCH EM_LOONGARCH +#define GEN_ELF_CLASS ELFCLASS64 #else #error "unsupported architecture" #endif diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index dd06770b43f1c2d0df7a941ade650d14485f39fd..d2812d98968df52def01c6663fcdf5f6403dfd08 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -1655,8 +1655,8 @@ static void print_bpf_prog_info(struct feat_fd *ff, FILE *fp) node = rb_entry(next, struct bpf_prog_info_node, rb_node); next = rb_next(&node->rb_node); - bpf_event__print_bpf_prog_info(&node->info_linear->info, - env, fp); + __bpf_event__print_bpf_prog_info(&node->info_linear->info, + env, fp); } up_read(&env->bpf_progs.lock); @@ -2927,7 +2927,7 @@ static int process_bpf_prog_info(struct feat_fd *ff, void *data __maybe_unused) /* after reading from file, translate offset to address */ bpf_program__bpil_offs_to_addr(info_linear); info_node->info_linear = info_linear; - perf_env__insert_bpf_prog_info(env, info_node); + __perf_env__insert_bpf_prog_info(env, info_node); } up_write(&env->bpf_progs.lock); @@ -2980,7 +2980,7 @@ static int process_bpf_btf(struct feat_fd *ff, void *data __maybe_unused) if (__do_read(ff, node->data, data_size)) goto out; - perf_env__insert_btf(env, node); + __perf_env__insert_btf(env, node); node = NULL; } diff --git a/tools/perf/util/syscalltbl.c b/tools/perf/util/syscalltbl.c index 03bd99d3be16f352cc7ae09d304340d9a8b2cb87..e441f12c7272274abea56c926274e22e50eada9b 100644 --- a/tools/perf/util/syscalltbl.c +++ b/tools/perf/util/syscalltbl.c @@ -34,6 +34,10 @@ static const char **syscalltbl_native = syscalltbl_powerpc_32; #include const int syscalltbl_native_max_id = SYSCALLTBL_ARM64_MAX_ID; static const char **syscalltbl_native = syscalltbl_arm64; +#elif defined(__loongarch__) +#include +const int syscalltbl_native_max_id = SYSCALLTBL_LOONGARCH_MAX_ID; +static const char **syscalltbl_native = syscalltbl_loongarch; #endif struct syscall { diff --git a/tools/scripts/Makefile.arch b/tools/scripts/Makefile.arch index b10b7a27c33fd16b10fa3d7fcfd7a5fa372f2d59..0b2af44c167fcc6a2b1faeecb935f32f9dc1db57 100644 --- a/tools/scripts/Makefile.arch +++ b/tools/scripts/Makefile.arch @@ -4,7 +4,8 @@ HOSTARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ -e /arm64/!s/arm.*/arm/ -e s/sa110/arm/ \ -e s/s390x/s390/ -e s/parisc64/parisc/ \ -e s/ppc.*/powerpc/ -e s/mips.*/mips/ \ - -e s/sh[234].*/sh/ -e s/aarch64.*/arm64/ ) + -e s/sh[234].*/sh/ -e s/aarch64.*/arm64/ \ + -e s/loongarch.*/loongarch/) ifndef ARCH ARCH := $(HOSTARCH) @@ -33,6 +34,15 @@ ifeq ($(ARCH),sh64) SRCARCH := sh endif +# Additional ARCH settings for loongarch +ifeq ($(ARCH),loongarch32) + SRCARCH := loongarch +endif + +ifeq ($(ARCH),loongarch64) + SRCARCH := loongarch +endif + LP64 := $(shell echo __LP64__ | ${CC} ${CFLAGS} -E -x c - | tail -n 1) ifeq ($(LP64), 1) IS_64_BIT := 1 diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 28d22265b8253399854c39d03ddf76115d1d1897..cbdc2839904ef565854dfd94557bca96ccfb86b6 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4611,6 +4611,7 @@ static size_t get_pprint_mapv_size(enum pprint_mapv_kind_t mapv_kind) #endif assert(0); + return 0; } static void set_pprint_mapv(enum pprint_mapv_kind_t mapv_kind, diff --git a/tools/testing/selftests/bpf/progs/pyperf180.c b/tools/testing/selftests/bpf/progs/pyperf180.c index c39f559d3100e8c32a87509e1101fc30fc73f2e8..42c4a8b62e3602d7a4ce7e2b529484a317ae2246 100644 --- a/tools/testing/selftests/bpf/progs/pyperf180.c +++ b/tools/testing/selftests/bpf/progs/pyperf180.c @@ -1,4 +1,26 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 Facebook #define STACK_MAX_LEN 180 + +/* llvm upstream commit at clang18 + * https://github.com/llvm/llvm-project/commit/1a2e77cf9e11dbf56b5720c607313a566eebb16e + * changed inlining behavior and caused compilation failure as some branch + * target distance exceeded 16bit representation which is the maximum for + * cpu v1/v2/v3. Macro __BPF_CPU_VERSION__ is later implemented in clang18 + * to specify which cpu version is used for compilation. So a smaller + * unroll_count can be set if __BPF_CPU_VERSION__ is less than 4, which + * reduced some branch target distances and resolved the compilation failure. + * + * To capture the case where a developer/ci uses clang18 but the corresponding + * repo checkpoint does not have __BPF_CPU_VERSION__, a smaller unroll_count + * will be set as well to prevent potential compilation failures. + */ +#ifdef __BPF_CPU_VERSION__ +#if __BPF_CPU_VERSION__ < 4 +#define UNROLL_COUNT 90 +#endif +#elif __clang_major__ == 18 +#define UNROLL_COUNT 90 +#endif + #include "pyperf.h" diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh index 5c7700212f75377f0f82444194fbcd0801f717ef..56761de1ca3b7d28df9ffa8dd018dc8b0ef8bc6a 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh @@ -121,6 +121,9 @@ h2_destroy() switch_create() { + local lanes_swp4 + local pg1_size + # pools # ----- @@ -171,7 +174,7 @@ switch_create() # assignment. tc qdisc replace dev $swp1 root handle 1: \ ets bands 8 strict 8 priomap 7 6 - __mlnx_qos -i $swp1 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null + dcb buffer set dev $swp1 prio-buffer all:0 1:1 # $swp2 # ----- @@ -209,8 +212,8 @@ switch_create() # the lossless prio into a buffer of its own. Don't bother with buffer # sizes though, there is not going to be any pressure in the "backward" # direction. - __mlnx_qos -i $swp3 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null - __mlnx_qos -i $swp3 --pfc=0,1,0,0,0,0,0,0 >/dev/null + dcb buffer set dev $swp3 prio-buffer all:0 1:1 + dcb pfc set dev $swp3 prio-pfc all:off 1:on # $swp4 # ----- @@ -226,11 +229,24 @@ switch_create() # Configure qdisc so that we can hand-tune headroom. tc qdisc replace dev $swp4 root handle 1: \ ets bands 8 strict 8 priomap 7 6 - __mlnx_qos -i $swp4 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null - __mlnx_qos -i $swp4 --pfc=0,1,0,0,0,0,0,0 >/dev/null + dcb buffer set dev $swp4 prio-buffer all:0 1:1 + dcb pfc set dev $swp4 prio-pfc all:off 1:on # PG0 will get autoconfigured to Xoff, give PG1 arbitrarily 100K, which # is (-2*MTU) about 80K of delay provision. - __mlnx_qos -i $swp4 --buffer_size=0,$_100KB,0,0,0,0,0,0 >/dev/null + pg1_size=$_100KB + + setup_wait_dev_with_timeout $swp4 + + lanes_swp4=$(ethtool $swp4 | grep 'Lanes:') + lanes_swp4=${lanes_swp4#*"Lanes: "} + + # 8-lane ports use two buffers among which the configured buffer + # is split, so double the size to get twice (20K + 80K). + if [[ $lanes_swp4 -eq 8 ]]; then + pg1_size=$((pg1_size * 2)) + fi + + dcb buffer set dev $swp4 buffer-size all:0 1:$pg1_size # bridges # ------- @@ -273,9 +289,9 @@ switch_destroy() # $swp4 # ----- - __mlnx_qos -i $swp4 --buffer_size=0,0,0,0,0,0,0,0 >/dev/null - __mlnx_qos -i $swp4 --pfc=0,0,0,0,0,0,0,0 >/dev/null - __mlnx_qos -i $swp4 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + dcb buffer set dev $swp4 buffer-size all:0 + dcb pfc set dev $swp4 prio-pfc all:off + dcb buffer set dev $swp4 prio-buffer all:0 tc qdisc del dev $swp4 root devlink_tc_bind_pool_th_restore $swp4 1 ingress @@ -288,8 +304,8 @@ switch_destroy() # $swp3 # ----- - __mlnx_qos -i $swp3 --pfc=0,0,0,0,0,0,0,0 >/dev/null - __mlnx_qos -i $swp3 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + dcb pfc set dev $swp3 prio-pfc all:off + dcb buffer set dev $swp3 prio-buffer all:0 tc qdisc del dev $swp3 root devlink_tc_bind_pool_th_restore $swp3 1 egress @@ -315,7 +331,7 @@ switch_destroy() # $swp1 # ----- - __mlnx_qos -i $swp1 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + dcb buffer set dev $swp1 prio-buffer all:0 tc qdisc del dev $swp1 root devlink_tc_bind_pool_th_restore $swp1 1 ingress diff --git a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/tc_flower.sh b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/tc_flower.sh index fb850e0ec8375f143c8da0459ee48ca778bb3a29..616d3581419ca043fc715f9d067341c89513f6eb 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/tc_flower.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/spectrum-2/tc_flower.sh @@ -10,7 +10,8 @@ lib_dir=$(dirname $0)/../../../../net/forwarding ALL_TESTS="single_mask_test identical_filters_test two_masks_test \ multiple_masks_test ctcam_edge_cases_test delta_simple_test \ delta_two_masks_one_key_test delta_simple_rehash_test \ - bloom_simple_test bloom_complex_test bloom_delta_test" + bloom_simple_test bloom_complex_test bloom_delta_test \ + max_erp_entries_test max_group_size_test" NUM_NETIFS=2 source $lib_dir/lib.sh source $lib_dir/tc_common.sh @@ -983,6 +984,109 @@ bloom_delta_test() log_test "bloom delta test ($tcflags)" } +max_erp_entries_test() +{ + # The number of eRP entries is limited. Once the maximum number of eRPs + # has been reached, filters cannot be added. This test verifies that + # when this limit is reached, inserstion fails without crashing. + + RET=0 + + local num_masks=32 + local num_regions=15 + local chain_failed + local mask_failed + local ret + + if [[ "$tcflags" != "skip_sw" ]]; then + return 0; + fi + + for ((i=1; i < $num_regions; i++)); do + for ((j=$num_masks; j >= 0; j--)); do + tc filter add dev $h2 ingress chain $i protocol ip \ + pref $i handle $j flower $tcflags \ + dst_ip 192.1.0.0/$j &> /dev/null + ret=$? + + if [ $ret -ne 0 ]; then + chain_failed=$i + mask_failed=$j + break 2 + fi + done + done + + # We expect to exceed the maximum number of eRP entries, so that + # insertion eventually fails. Otherwise, the test should be adjusted to + # add more filters. + check_fail $ret "expected to exceed number of eRP entries" + + for ((; i >= 1; i--)); do + for ((j=0; j <= $num_masks; j++)); do + tc filter del dev $h2 ingress chain $i protocol ip \ + pref $i handle $j flower &> /dev/null + done + done + + log_test "max eRP entries test ($tcflags). " \ + "max chain $chain_failed, mask $mask_failed" +} + +max_group_size_test() +{ + # The number of ACLs in an ACL group is limited. Once the maximum + # number of ACLs has been reached, filters cannot be added. This test + # verifies that when this limit is reached, insertion fails without + # crashing. + + RET=0 + + local num_acls=32 + local max_size + local ret + + if [[ "$tcflags" != "skip_sw" ]]; then + return 0; + fi + + for ((i=1; i < $num_acls; i++)); do + if [[ $(( i % 2 )) == 1 ]]; then + tc filter add dev $h2 ingress pref $i proto ipv4 \ + flower $tcflags dst_ip 198.51.100.1/32 \ + ip_proto tcp tcp_flags 0x01/0x01 \ + action drop &> /dev/null + else + tc filter add dev $h2 ingress pref $i proto ipv6 \ + flower $tcflags dst_ip 2001:db8:1::1/128 \ + action drop &> /dev/null + fi + + ret=$? + [[ $ret -ne 0 ]] && max_size=$((i - 1)) && break + done + + # We expect to exceed the maximum number of ACLs in a group, so that + # insertion eventually fails. Otherwise, the test should be adjusted to + # add more filters. + check_fail $ret "expected to exceed number of ACLs in a group" + + for ((; i >= 1; i--)); do + if [[ $(( i % 2 )) == 1 ]]; then + tc filter del dev $h2 ingress pref $i proto ipv4 \ + flower $tcflags dst_ip 198.51.100.1/32 \ + ip_proto tcp tcp_flags 0x01/0x01 \ + action drop &> /dev/null + else + tc filter del dev $h2 ingress pref $i proto ipv6 \ + flower $tcflags dst_ip 2001:db8:1::1/128 \ + action drop &> /dev/null + fi + done + + log_test "max ACL group size test ($tcflags). max size $max_size" +} + setup_prepare() { h1=${NETIFS[p1]} diff --git a/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh b/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh index 1b08e042cf942a126626bbd3cad88152633f68a9..185b02d2d4cd14720d9084afc94b3b1ceb89c4b6 100755 --- a/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh +++ b/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh @@ -269,6 +269,7 @@ for port in 0 1; do echo 1 > $NSIM_DEV_SYS/new_port fi NSIM_NETDEV=`get_netdev_name old_netdevs` + ifconfig $NSIM_NETDEV up msg="new NIC device created" exp0=( 0 0 0 0 ) @@ -430,6 +431,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up overflow_table0 "overflow NIC table" @@ -487,6 +489,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up overflow_table0 "overflow NIC table" @@ -543,6 +546,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up overflow_table0 "destroy NIC" @@ -572,6 +576,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up msg="create VxLANs v6" @@ -632,6 +637,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up echo 110 > $NSIM_DEV_DFS/ports/$port/udp_ports_inject_error @@ -687,6 +693,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up msg="create VxLANs v6" @@ -746,6 +753,7 @@ for port in 0 1; do fi echo $port > $NSIM_DEV_SYS/new_port + NSIM_NETDEV=`get_netdev_name old_netdevs` ifconfig $NSIM_NETDEV up msg="create VxLANs v6" @@ -876,6 +884,7 @@ msg="re-add a port" echo 2 > $NSIM_DEV_SYS/del_port echo 2 > $NSIM_DEV_SYS/new_port +NSIM_NETDEV=`get_netdev_name old_netdevs` check_tables msg="replace VxLAN in overflow table" diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 413a7b9f3c4d3f8848f9fbd72b99d7162d3aef08..f725d83d3ede660c46cf888951d102da23eb2fe7 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -125,6 +125,8 @@ struct seccomp_data { # define __NR_seccomp 277 # elif defined(__csky__) # define __NR_seccomp 277 +# elif defined(__loongarch__) +# define __NR_seccomp 277 # elif defined(__hppa__) # define __NR_seccomp 338 # elif defined(__powerpc__) @@ -1730,6 +1732,10 @@ TEST_F(TRACE_poke, getpid_runs_normally) NT_ARM_SYSTEM_CALL, &__v)); \ } while (0) # define SYSCALL_RET(_regs) (_regs).regs[0] +#elif defined(__loongarch__) +# define ARCH_REGS struct user_pt_regs +# define SYSCALL_NUM(_regs) (_regs).regs[11] +# define SYSCALL_RET(_regs) (_regs).regs[4] #elif defined(__riscv) && __riscv_xlen == 64 # define ARCH_REGS struct user_regs_struct # define SYSCALL_NUM(_regs) (_regs).a7 diff --git a/virt/kvm/irqchip.c b/virt/kvm/irqchip.c index 58e4f88b2b9fb4ecd0765f6e653f8dc14b5e0423..77e79f70645eb7f127bba96fdfe8d786acd32d7e 100644 --- a/virt/kvm/irqchip.c +++ b/virt/kvm/irqchip.c @@ -19,6 +19,10 @@ #include #include "irq.h" +#if defined(CONFIG_CPU_LOONGSON64) +#include "ls_irq.h" +#endif + int kvm_irq_map_gsi(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *entries, int gsi) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 356fd5d1a4285a8b61c2dced37c7876ee50d8360..b7638c3c9eb7d88e8efcd6b07efa3db07c760516 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1008,9 +1008,9 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) */ static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) { - unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot); + unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot); - memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT); + memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT); if (!memslot->dirty_bitmap) return -ENOMEM;